fix: doc links and delete replication permissions enforcement (#11285)

Fixes: #11272

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,9 +10,10 @@
 ## Types of changes
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
+- [ ] Optimization (provides speedup with no functional changes)
 - [ ] Breaking change (fix or feature that would cause existing functionality to change)
 
 ## Checklist:
 - [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
-- [ ] Documentation needed
-- [ ] Unit tests needed
+- [ ] Documentation updated
+- [ ] Unit tests added/updated

.github/workflows/go.yml

@@ -26,7 +26,6 @@ jobs:
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          MINIO_CI_CD: 1
         run: |
           go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
           go test -v --timeout 50m ./...
@@ -35,13 +34,12 @@ jobs:
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          MINIO_CI_CD: 1
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           sudo apt-get install devscripts shellcheck
           nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
-          curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-linux.amd64-${nancy_version} && chmod +x nancy
+          curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-${nancy_version}-linux-amd64 && chmod +x nancy
           go list -m all | ./nancy sleuth
           make
           diff -au <(gofmt -s -d cmd) <(printf "")

.golangci.yml

@@ -23,5 +23,11 @@ issues:
   exclude:
       - should have a package comment
       - error strings should not be capitalized or end with punctuation or a newline
+
+run:
+  skip-dirs:
+    - pkg/rpc
+    - pkg/argon2
+
 service:
   golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly

Dockerfile

@@ -11,10 +11,12 @@ RUN  \
      git clone https://github.com/minio/minio && cd minio && \
      git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
 
-FROM alpine:3.12
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
 
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
     MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
@@ -22,11 +24,14 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 EXPOSE 9000
 
 COPY --from=builder /go/bin/minio /usr/bin/minio
-COPY --from=builder /go/minio/CREDITS /third_party/
+COPY --from=builder /go/minio/CREDITS /licenses/CREDITS
+COPY --from=builder /go/minio/LICENSE /licenses/LICENSE
 COPY --from=builder /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     microdnf update --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+     microdnf clean all && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]

Dockerfile.arm.release

@@ -1,34 +0,0 @@
-FROM multiarch/qemu-user-static:x86_64-arm as qemu
-FROM arm32v7/alpine:3.12
-
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-COPY --from=qemu /usr/bin/qemu-arm-static /usr/bin
-
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
-    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
-
-RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio.minisig -o /usr/bin/minio.minisig && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS
-
-EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Dockerfile.arm64.release

@@ -1,35 +0,0 @@
-FROM multiarch/qemu-user-static:x86_64-aarch64 as qemu
-FROM arm64v8/alpine:3.12
-
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin
-
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
-    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
-
-RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio.minisig -o /usr/bin/minio.minisig && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
-     /usr/bin/verify-minio.sh
-
-EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Dockerfile.cicd

@@ -0,0 +1,42 @@
+FROM golang:1.15-alpine as builder
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+    apk add --no-cache git && \
+    git clone https://github.com/minio/minio && cd minio && \
+    git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
+
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
+
+ARG TARGETARCH
+
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
+
+EXPOSE 9000
+
+COPY --from=builder /go/bin/minio /usr/bin/minio
+COPY --from=builder /go/minio/CREDITS /licenses/CREDITS
+COPY --from=builder /go/minio/LICENSE /licenses/LICENSE
+COPY --from=builder /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
+
+RUN  \
+    microdnf update --nodocs && \
+    microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+    microdnf clean all
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio", "server", "/data"]

Dockerfile.dev

@@ -1,22 +1,28 @@
-FROM alpine:3.12
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
+
+ARG TARGETARCH
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 COPY minio /usr/bin/
-COPY CREDITS /third_party/
+COPY CREDITS /licenses/CREDITS
+COPY LICENSE /licenses/LICENSE
 
-ENV MINIO_UPDATE off
-ENV MINIO_ACCESS_KEY_FILE=access_key \
+ENV MINIO_UPDATE=off \
+    MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
     MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
-RUN \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     chmod +x /usr/bin/minio  && \
-     chmod +x /usr/bin/docker-entrypoint.sh
+RUN  \
+    microdnf update --nodocs && \
+    microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+    microdnf clean all && \
+    chmod +x /usr/bin/minio  && \
+    chmod +x /usr/bin/docker-entrypoint.sh
 
 EXPOSE 9000
 

Dockerfile.dev.browser

@@ -1,4 +1,4 @@
-FROM ubuntu
+FROM ubuntu:20.04
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 

Dockerfile.ppc64le.release

@@ -1,34 +0,0 @@
-FROM multiarch/qemu-user-static:x86_64-ppc64le as qemu
-FROM ppc64le/alpine:3.12
-
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-COPY --from=qemu /usr/bin/qemu-ppc64le-static /usr/bin
-
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
-    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
-
-RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio.minisig -o /usr/bin/minio.minisig && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
-     /usr/bin/verify-minio.sh
-
-EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Dockerfile.release

@@ -1,26 +1,40 @@
-FROM alpine:3.12
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
 
-LABEL maintainer="MinIO Inc <dev@min.io>"
+ARG TARGETARCH
+
+LABEL name="MinIO" \
+      vendor="MinIO Inc <dev@min.io>" \
+      maintainer="MinIO Inc <dev@min.io>" \
+      version="RELEASE.2021-01-08T21-18-21Z" \
+      release="RELEASE.2021-01-08T21-18-21Z" \
+      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
+      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
 
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
     MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
+COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+COPY CREDITS /licenses/CREDITS
+COPY LICENSE /licenses/LICENSE
+
 RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio.minisig -o /usr/bin/minio.minisig && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     microdnf update --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+     microdnf install minisign --nodocs && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio -o /usr/bin/minio && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio.sha256sum -o /usr/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio.minisig -o /usr/bin/minio.minisig && \
+     microdnf clean all && \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
      /usr/bin/verify-minio.sh
 
 EXPOSE 9000

Dockerfile.s390x.release

@@ -1,34 +0,0 @@
-FROM multiarch/qemu-user-static:x86_64-s390x as qemu
-FROM s390x/alpine:3.12
-
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-COPY --from=qemu /usr/bin/qemu-s390x-static /usr/bin
-
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
-    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
-
-RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio.minisig -o /usr/bin/minio.minisig && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
-     /usr/bin/verify-minio.sh
-
-EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Makefile

@@ -17,9 +17,9 @@ checks:
 getdeps:
 	@mkdir -p ${GOPATH}/bin
 	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
-	@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && GO111MODULE=off go get github.com/quasilyte/go-ruleguard/...)
-	@which msgp 1>/dev/null || (echo "Installing msgp" && GO111MODULE=off go get github.com/tinylib/msgp)
-	@which stringer 1>/dev/null || (echo "Installing stringer" && GO111MODULE=off go get golang.org/x/tools/cmd/stringer)
+	@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && go get github.com/quasilyte/go-ruleguard/cmd/ruleguard@v0.2.1)
+	@which msgp 1>/dev/null || (echo "Installing msgp" && go get github.com/tinylib/msgp@v1.1.3)
+	@which stringer 1>/dev/null || (echo "Installing stringer" && go get golang.org/x/tools/cmd/stringer)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
@@ -57,7 +57,7 @@ test-race: verifiers build
 # Verify minio binary
 verify:
 	@echo "Verifying build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 # Verify healing of disks with minio binary
@@ -71,6 +71,10 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
+hotfix: LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
+    sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#'))
+hotfix: install
+
 docker: checks
 	@echo "Building minio docker image '$(TAG)'"
 	@GOOS=linux GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null

README.md

@@ -5,80 +5,190 @@
 
 MinIO is a High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
 
-## Docker Container
-### Stable
-```
-docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  minio/minio server /data
+This README provides quickstart instructions on running MinIO on baremetal hardware, including Docker-based installations. For Kubernetes environments, 
+use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md). 
+
+**Table of Contents**
+
+- [Docker Installation](#docker-installation)
+- [macOS Installation](#macos)
+- [GNU/Linux Installation](#gnulinux)
+- [Windows Installation](#microsoft-windows)
+- [FreeBSD Installation](#freebsd)
+- [Source Installation](#install-from-source)
+- [Deployment Recommendations](#deployment-recommendations)
+- [Test Connectivity to MinIO](#test-minio-connectivity)
+- [Upgrade MinIO](#upgrading-minio)
+- [Explore Further](#explore-further)
+- [Contribute to MinIO](#contribute-to-minio-project)
+- [License](#license)
+
+# Docker Installation
+
+Use the following commands to run a standalone MinIO server on a Docker container. 
+
+Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+for more complete documentation.
+
+## Stable
+
+Run the following command to run the latest stable image of MinIO on a Docker container using an ephemeral data volume: 
+
+```sh
+docker run -p 9000:9000 minio/minio server /data
 ```
 
-### Edge
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+
+> NOTE: To deploy MinIO on Docker with persistent storage, you must map local persistent directories from the host OS to the container using the
+  `docker -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the Docker container.
+
+## Edge
+
+Run the following command to run the bleeding-edge image of MinIO on a Docker container using an ephemeral data volume:
+
 ```
-docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  minio/minio:edge server /data
+docker run -p 9000:9000 minio/minio:edge server /data
 ```
 
-> NOTE: Docker will not display the default keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use default keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+
+> NOTE: To deploy MinIO on Docker with persistent storage, you must map local persistent directories from the host OS to the container using the
+  `docker -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the Docker container.
+
+# macOS
+
+Use the following commands to run a standalone MinIO server on macOS.
+
+Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+for more complete documentation.
+
+## Homebrew (recommended)
+
+Run the following command to install the latest stable MinIO package using [Homebrew](https://brew.sh/). Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
 
-## macOS
-### Homebrew (recommended)
-Install minio packages using [Homebrew](https://brew.sh/)
 ```sh
 brew install minio/stable/minio
 minio server /data
 ```
 
 > NOTE: If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
+
 ```sh
 brew uninstall minio
 brew install minio/stable/minio
 ```
 
-### Binary Download
-| Platform    | Architecture | URL                                                       |
-| ----------  | --------     | ------                                                    |
-| Apple macOS | 64-bit Intel | https://dl.min.io/server/minio/release/darwin-amd64/minio |
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+## Binary Download
+
+Use the following command to download and run a standalone MinIO server on macOS. Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
+
 ```sh
-chmod 755 minio
+wget https://dl.min.io/server/minio/release/darwin-amd64/minio
+chmod +x minio
 ./minio server /data
 ```
 
-## GNU/Linux
-### Binary Download
-| Platform   | Architecture | URL                                                      |
-| ---------- | --------     | ------                                                   |
-| GNU/Linux  | 64-bit Intel | https://dl.min.io/server/minio/release/linux-amd64/minio |
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+
+# GNU/Linux
+
+Use the following command to run a standalone MinIO server on Linux hosts running 64-bit Intel/AMD architectures. Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
+
 ```sh
 wget https://dl.min.io/server/minio/release/linux-amd64/minio
 chmod +x minio
 ./minio server /data
 ```
 
-| Platform   | Architecture | URL                                                        |
-| ---------- | --------     | ------                                                     |
-| GNU/Linux  | ppc64le      | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
+Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
+
+The following table lists supported architectures. Replace the `wget` URL with the architecture for your Linux host.
+
+| Architecture                   | URL                                                        |
+| --------                       | ------                                                     |
+| 64-bit Intel/AMD               | https://dl.min.io/server/minio/release/linux-amd64/minio   |
+| 64-bit ARM                     | https://dl.min.io/server/minio/release/linux-arm64/minio   |
+| 64-bit PowerPC LE (ppc64le)    | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
+| IBM Z-Series (S390X)           | https://dl.min.io/server/minio/release/linux-s390x/minio   |
+
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+for more complete documentation.
+
+# Microsoft Windows
+
+To run MinIO on 64-bit Windows hosts, download the MinIO executable from the following URL:
+
 ```sh
-wget https://dl.min.io/server/minio/release/linux-ppc64le/minio
-chmod +x minio
-./minio server /data
+https://dl.min.io/server/minio/release/windows-amd64/minio.exe
 ```
 
-## Microsoft Windows
-### Binary Download
-| Platform          | Architecture | URL                                                            |
-| ----------        | --------     | ------                                                         |
-| Microsoft Windows | 64-bit       | https://dl.min.io/server/minio/release/windows-amd64/minio.exe |
+Use the following command to run a standalone MinIO server on the Windows host. Replace ``D:\`` with the path to the drive or directory in which you want MinIO to store data. You must change the terminal or powershell directory to the location of the ``minio.exe`` executable, *or* add the path to that directory to the system ``$PATH``:
+
 ```sh
-minio.exe server D:\Photos
+minio.exe server D:\
 ```
 
-## FreeBSD
-### Port
-Install minio packages using [pkg](https://github.com/freebsd/pkg), MinIO doesn't officially build FreeBSD binaries but is maintained by FreeBSD upstream [here](https://www.freshports.org/www/minio).
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+for more complete documentation.
+
+# FreeBSD
+
+MinIO does not provide an official FreeBSD binary. However, FreeBSD maintains an [upstream release](https://www.freshports.org/www/minio) using [pkg](https://github.com/freebsd/pkg):
 
 ```sh
 pkg install minio
@@ -87,13 +197,32 @@ sysrc minio_disks=/home/user/Photos
 service minio start
 ```
 
-## Install from Source
-Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)
+# Install from Source
+
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio
 ```
 
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+for more complete documentation. 
+
+MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments. 
+
+# Deployment Recommendations
+
 ## Allow port access for Firewalls
 
 By default MinIO uses the port 9000 to listen for incoming connections. If your platform blocks the port by default, you may need to enable access to the port.
@@ -149,20 +278,22 @@ iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
 service iptables restart
 ```
 
+## Pre-existing data
+When deployed on a single drive, MinIO server lets clients access any pre-existing data in the data directory. For example, if MinIO is started with the command  `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.
+
+The above statement is also valid for all gateway backends.
+
+# Test MinIO Connectivity
+
 ## Test using MinIO Browser
-MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully.
+MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure your server has started successfully.
 
 ![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.png?raw=true)
 
 ## Test using MinIO Client `mc`
 `mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/docs/minio-client-quickstart-guide) for further instructions.
 
-## Pre-existing data
-When deployed on a single drive, MinIO server lets clients access any pre-existing data in the data directory. For example, if MinIO is started with the command  `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.
-
-The above statement is also valid for all gateway backends.
-
-## Upgrading MinIO
+# Upgrading MinIO
 MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster simultaneously and restart them, as shown in the following command from the MinIO client (mc):
 
 ```
@@ -171,7 +302,7 @@ mc admin update <minio alias, e.g., myminio>
 
 > NOTE: some releases might not allow rolling upgrades, this is always called out in the release notes and it is generally advised to read release notes before upgrading. In such a situation `mc admin update` is the recommended upgrading mechanism to upgrade all servers at once.
 
-### Important things to remember during MinIO upgrades
+## Important things to remember during MinIO upgrades
 
 - `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
 - `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
@@ -181,7 +312,7 @@ mc admin update <minio alias, e.g., myminio>
 - If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
 - If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
 
-## Explore Further
+# Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
 - [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
 - [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio)
@@ -189,8 +320,8 @@ mc admin update <minio alias, e.g., myminio>
 - [Use `minio-go` SDK with MinIO Server](https://docs.min.io/docs/golang-client-quickstart-guide)
 - [The MinIO documentation website](https://docs.min.io)
 
-## Contribute to MinIO Project
+# Contribute to MinIO Project
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
-## License
+# License
 Use of MinIO is governed by the Apache 2.0 License found at [LICENSE](https://github.com/minio/minio/blob/master/LICENSE).

README_zh_CN.md

@@ -8,16 +8,16 @@ MinIO是一个非常轻量的服务,可以很简单的和其他应用的结合
 ### 稳定版
 ```
 docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
   minio/minio server /data
 ```
 
 ### 尝鲜版
 ```
 docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
   minio/minio:edge server /data
 ```
 

VULNERABILITY_REPORT.md

@@ -0,0 +1,39 @@
+## Vulnerability Management Policy
+
+This document formally describes the process of addressing and managing a
+reported vulnerability that has been found in the MinIO server code base,
+any directly connected ecosystem component or a direct / indirect dependency
+of the code base.
+
+### Scope
+
+The vulnerability management policy described in this document covers the
+process of investigating, assessing and resolving a vulnerability report
+opened by a MinIO employee or an external third party.
+
+Therefore, it lists pre-conditions and actions that should be performed to
+resolve and fix a reported vulnerability.
+
+### Vulnerability Management Process
+
+The vulnerability management process requires that the vulnerability report
+contains the following information:
+
+ - The project / component that contains the reported vulnerability.
+ - A description of the vulnerability. In particular, the type of the
+   reported vulnerability and how it might be exploited. Alternatively,
+   a well-established vulnerability identifier, e.g. CVE number, can be
+   used instead.
+
+Based on the description mentioned above, a MinIO engineer or security team
+member investigates:
+
+ - Whether the reported vulnerability exists.
+ - The conditions that are required such that the vulnerability can be exploited.
+ - The steps required to fix the vulnerability.
+
+In general, if the vulnerability exists in one of the MinIO code bases
+itself - not in a code dependency - then MinIO will, if possible, fix
+the vulnerability or implement reasonable countermeasures such that the
+vulnerability cannot be exploited anymore.
+

browser/app/js/uploads/__tests__/actions.test.js

@@ -138,9 +138,10 @@ describe("Uploads actions", () => {
         objects: { currentPrefix: "pre1/" }
       })
       store.dispatch(uploadsActions.uploadFile(file))
+      const objectPath = encodeURIComponent("pre1/file1")
       expect(open).toHaveBeenCalledWith(
         "PUT",
-        "https://localhost:8080/upload/test1/pre1/file1",
+        "https://localhost:8080/upload/test1/" + objectPath,
         true
       )
       expect(send).toHaveBeenCalledWith(file)

browser/app/js/uploads/actions.js

@@ -94,7 +94,7 @@ export const uploadFile = file => {
       _filePath = _filePath.substring(1)
     }
     const filePath = _filePath
-    const objectName = `${currentPrefix}${filePath}`
+    const objectName = encodeURIComponent(`${currentPrefix}${filePath}`)
     const uploadUrl = `${
       window.location.origin
     }${minioBrowserPrefix}/upload/${currentBucket}/${objectName}`

browser/app/js/web.js

@@ -21,7 +21,7 @@ import storage from 'local-storage-fallback'
 
 class Web {
   constructor(endpoint) {
-    const namespace = 'Web'
+    const namespace = 'web'
     this.JSONrpc = new JSONrpc({
       endpoint,
       namespace

browser/package-lock.json

@@ -2271,7 +2271,9 @@
     "asap": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
-      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
+      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=",
+      "dev": true,
+      "optional": true
     },
     "asn1": {
       "version": "0.2.4",
@@ -2283,19 +2285,20 @@
       }
     },
     "asn1.js": {
-      "version": "4.10.1",
-      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz",
-      "integrity": "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==",
+      "version": "5.4.1",
+      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz",
+      "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==",
       "requires": {
         "bn.js": "^4.0.0",
         "inherits": "^2.0.1",
-        "minimalistic-assert": "^1.0.0"
+        "minimalistic-assert": "^1.0.0",
+        "safer-buffer": "^2.1.0"
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -3541,9 +3544,9 @@
       }
     },
     "base64-js": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
-      "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
     },
     "batch": {
       "version": "0.6.1",
@@ -3574,13 +3577,12 @@
     "bluebird": {
       "version": "3.7.2",
       "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
-      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==",
-      "dev": true
+      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
     },
     "bn.js": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.1.tgz",
-      "integrity": "sha512-IUTD/REb78Z2eodka1QZyyEk66pciRcP6Sroka0aI3tG/iwIdYLrBD62RsubR7vqdt3WyX8p4jxeatzmRSphtA=="
+      "version": "5.1.3",
+      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.3.tgz",
+      "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ=="
     },
     "body-parser": {
       "version": "1.19.0",
@@ -3754,31 +3756,24 @@
       }
     },
     "browserify-rsa": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
-      "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz",
+      "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==",
       "requires": {
-        "bn.js": "^4.1.0",
+        "bn.js": "^5.0.0",
         "randombytes": "^2.0.1"
-      },
-      "dependencies": {
-        "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
-        }
       }
     },
     "browserify-sign": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.0.tgz",
-      "integrity": "sha512-hEZC1KEeYuoHRqhGhTy6gWrpJA3ZDjFWv0DE61643ZnOXAKJb3u7yWcrU0mMc9SwAqK1n7myPGndkp0dFG7NFA==",
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz",
+      "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==",
       "requires": {
         "bn.js": "^5.1.1",
         "browserify-rsa": "^4.0.1",
         "create-hash": "^1.2.0",
         "create-hmac": "^1.1.7",
-        "elliptic": "^6.5.2",
+        "elliptic": "^6.5.3",
         "inherits": "^2.0.4",
         "parse-asn1": "^5.1.5",
         "readable-stream": "^3.6.0",
@@ -4022,9 +4017,9 @@
       }
     },
     "chokidar": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.0.tgz",
-      "integrity": "sha512-aXAaho2VJtisB/1fg1+3nlLJqGOuewTzQpd/Tz0yTg2R0e4IGtshYvtjowyEumcBv2z+y4+kc75Mz7j5xJskcQ==",
+      "version": "3.4.3",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.3.tgz",
+      "integrity": "sha512-DtM3g7juCXQxFVSNPNByEC2+NImtBuxQQvWlHunpJIS5Ocr0lG306cC7FCi7cEA0fzmybPUIl4txBIobk1gGOQ==",
       "optional": true,
       "requires": {
         "anymatch": "~3.1.1",
@@ -4034,7 +4029,7 @@
         "is-binary-path": "~2.1.0",
         "is-glob": "~4.0.1",
         "normalize-path": "~3.0.0",
-        "readdirp": "~3.4.0"
+        "readdirp": "~3.5.0"
       },
       "dependencies": {
         "anymatch": {
@@ -4048,9 +4043,9 @@
           }
         },
         "binary-extensions": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
-          "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==",
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz",
+          "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
           "optional": true
         },
         "braces": {
@@ -4123,9 +4118,9 @@
           "optional": true
         },
         "readdirp": {
-          "version": "3.4.0",
-          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz",
-          "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==",
+          "version": "3.5.0",
+          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz",
+          "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==",
           "optional": true,
           "requires": {
             "picomatch": "^2.2.1"
@@ -4143,9 +4138,9 @@
       }
     },
     "chownr": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
-      "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g=="
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
     },
     "chrome-trace-event": {
       "version": "1.0.2",
@@ -4713,18 +4708,18 @@
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
     "create-ecdh": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz",
-      "integrity": "sha512-GbEHQPMOswGpKXM9kCWVrremUcBmjteUaQ01T9rkKCPDXfUHX0IoP9LpHYo2NPFampa4e+/pFDc3jQdxrxQLaw==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
+      "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==",
       "requires": {
         "bn.js": "^4.1.0",
-        "elliptic": "^6.0.0"
+        "elliptic": "^6.5.3"
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -4925,9 +4920,9 @@
       }
     },
     "cyclist": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-0.2.2.tgz",
-      "integrity": "sha1-GzN5LhHpFKL9bW7WRHRkRE5fpkA="
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz",
+      "integrity": "sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk="
     },
     "dashdash": {
       "version": "1.14.1",
@@ -5242,9 +5237,9 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -5420,9 +5415,9 @@
       }
     },
     "duplexify": {
-      "version": "3.6.1",
-      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.1.tgz",
-      "integrity": "sha512-vM58DwdnKmty+FSPzT14K9JXb90H+j5emaR4KYbr2KTIz00WHGbWOe5ghQTx233ZCLZtrGDALzKwcjEtSt35mA==",
+      "version": "3.7.1",
+      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
+      "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==",
       "requires": {
         "end-of-stream": "^1.0.0",
         "inherits": "^2.0.1",
@@ -5471,9 +5466,9 @@
       "dev": true
     },
     "elliptic": {
-      "version": "6.5.2",
-      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.2.tgz",
-      "integrity": "sha512-f4x70okzZbIQl/NSRLkI/+tteV/9WqL98zx+SQ69KbXxmVrmjwsNUPn/gYJJ0sHvEak24cZgHIPegRePAtA/xw==",
+      "version": "6.5.3",
+      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz",
+      "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==",
       "requires": {
         "bn.js": "^4.4.0",
         "brorand": "^1.0.1",
@@ -5485,9 +5480,9 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -5509,14 +5504,6 @@
       "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=",
       "dev": true
     },
-    "encoding": {
-      "version": "0.1.12",
-      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
-      "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=",
-      "requires": {
-        "iconv-lite": "~0.4.13"
-      }
-    },
     "end-of-stream": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
@@ -5526,9 +5513,9 @@
       }
     },
     "enhanced-resolve": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.1.1.tgz",
-      "integrity": "sha512-98p2zE+rL7/g/DzMHMTF4zZlCgeVdJ7yr6xzEpJRYwFYrGi9ANdn5DnJURg6RpBkyk60XYDnWIv51VfIhfNGuA==",
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.3.0.tgz",
+      "integrity": "sha512-3e87LvavsdxyoCfGusJnrZ5G8SLPOFeHSNpZI/ATL9a5leXo2k0w6MKnbqhdBad9qTobSfB20Ld7UmgoNbAZkQ==",
       "requires": {
         "graceful-fs": "^4.1.2",
         "memory-fs": "^0.5.0",
@@ -5886,9 +5873,9 @@
       },
       "dependencies": {
         "lodash": {
-          "version": "4.17.15",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
-          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "version": "4.17.20",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+          "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
           "dev": true
         },
         "react-is": {
@@ -6162,11 +6149,18 @@
       "dev": true
     },
     "esrecurse": {
-      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz",
-      "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==",
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+      "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
       "requires": {
-        "estraverse": "^4.1.0"
+        "estraverse": "^5.2.0"
+      },
+      "dependencies": {
+        "estraverse": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+          "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ=="
+        }
       }
     },
     "estraverse": {
@@ -6193,9 +6187,9 @@
       "dev": true
     },
     "events": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/events/-/events-3.1.0.tgz",
-      "integrity": "sha512-Rv+u8MLHNOdMjTAFeT3nCjHn2aGlx435FP/sDHNaRhDEMwyI/aB22Kj2qIN8R0cw3z28psEQLYwxVKLsKrMgWg=="
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.2.0.tgz",
+      "integrity": "sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg=="
     },
     "eventsource": {
       "version": "1.0.7",
@@ -6722,27 +6716,6 @@
         "bser": "2.1.1"
       }
     },
-    "fbjs": {
-      "version": "0.8.16",
-      "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.8.16.tgz",
-      "integrity": "sha1-XmdDL1UNxBtXK/VYR7ispk5TN9s=",
-      "requires": {
-        "core-js": "^1.0.0",
-        "isomorphic-fetch": "^2.1.1",
-        "loose-envify": "^1.0.0",
-        "object-assign": "^4.1.0",
-        "promise": "^7.1.1",
-        "setimmediate": "^1.0.5",
-        "ua-parser-js": "^0.7.9"
-      },
-      "dependencies": {
-        "core-js": {
-          "version": "1.2.7",
-          "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz",
-          "integrity": "sha1-ZSKUwUZR2yj6k70tX/KYOk8IxjY="
-        }
-      }
-    },
     "figgy-pudding": {
       "version": "3.5.2",
       "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz",
@@ -7013,12 +6986,12 @@
       }
     },
     "flush-write-stream": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.0.3.tgz",
-      "integrity": "sha512-calZMC10u0FMUqoiunI2AiGIIUtUIvifNwkHhNupZH4cbNnW1Itkoh/Nf5HFYmDrwWPjrUxpkZT0KhuCq0jmGw==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz",
+      "integrity": "sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==",
       "requires": {
-        "inherits": "^2.0.1",
-        "readable-stream": "^2.0.4"
+        "inherits": "^2.0.3",
+        "readable-stream": "^2.3.6"
       }
     },
     "follow-redirects": {
@@ -8376,9 +8349,9 @@
           }
         },
         "lodash": {
-          "version": "4.17.15",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
-          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "version": "4.17.20",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+          "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
           "dev": true
         },
         "tapable": {
@@ -8651,14 +8624,6 @@
       "resolved": "https://registry.npmjs.org/humanize/-/humanize-0.0.9.tgz",
       "integrity": "sha1-GZT/rs3+nEQe0r2sdFK3u0yeQaQ="
     },
-    "iconv-lite": {
-      "version": "0.4.23",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz",
-      "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==",
-      "requires": {
-        "safer-buffer": ">= 2.1.2 < 3"
-      }
-    },
     "icss-utils": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-4.1.1.tgz",
@@ -8677,9 +8642,9 @@
       }
     },
     "ieee754": {
-      "version": "1.1.13",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
-      "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
     },
     "iferr": {
       "version": "0.1.5",
@@ -8746,9 +8711,9 @@
       "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
     },
     "ini": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
-      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
       "dev": true
     },
     "internal-ip": {
@@ -9013,7 +8978,8 @@
     "is-stream": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
-      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
+      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=",
+      "dev": true
     },
     "is-subset": {
       "version": "0.1.1",
@@ -9068,15 +9034,6 @@
       "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
       "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
     },
-    "isomorphic-fetch": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz",
-      "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=",
-      "requires": {
-        "node-fetch": "^1.0.1",
-        "whatwg-fetch": ">=0.10.0"
-      }
-    },
     "isstream": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
@@ -10573,9 +10530,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.14",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz",
-      "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==",
+      "version": "4.17.20",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+      "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
       "dev": true
     },
     "lodash._baseisequal": {
@@ -10682,6 +10639,14 @@
         "tslib": "^1.10.0"
       }
     },
+    "lru-cache": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+      "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+      "requires": {
+        "yallist": "^3.0.2"
+      }
+    },
     "make-dir": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz",
@@ -10868,9 +10833,9 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -10993,6 +10958,23 @@
         }
       }
     },
+    "mississippi": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz",
+      "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==",
+      "requires": {
+        "concat-stream": "^1.5.0",
+        "duplexify": "^3.4.2",
+        "end-of-stream": "^1.1.0",
+        "flush-write-stream": "^1.0.0",
+        "from2": "^2.1.0",
+        "parallel-transform": "^1.1.0",
+        "pump": "^3.0.0",
+        "pumpify": "^1.3.3",
+        "stream-each": "^1.1.0",
+        "through2": "^2.0.0"
+      }
+    },
     "mixin-deep": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
@@ -11165,19 +11147,10 @@
         "tslib": "^1.10.0"
       }
     },
-    "node-fetch": {
-      "version": "1.6.3",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.6.3.tgz",
-      "integrity": "sha1-3CNO3WSJmC1Y6PDbT2lQKavNjAQ=",
-      "requires": {
-        "encoding": "^0.1.11",
-        "is-stream": "^1.0.1"
-      }
-    },
     "node-forge": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.0.tgz",
-      "integrity": "sha512-7ASaDa3pD+lJ3WvXFsxekJQelBKRpne+GOVbLbtHYdd7pFspyeuJHnWfLplGf3SwKGbfs/aYl5V/JCIaHVUKKQ==",
+      "version": "0.10.0",
+      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz",
+      "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==",
       "dev": true
     },
     "node-int64": {
@@ -11768,11 +11741,11 @@
       "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
     },
     "parallel-transform": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.1.0.tgz",
-      "integrity": "sha1-1BDwZbBdojCB/NEPKIVMKb2jOwY=",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz",
+      "integrity": "sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==",
       "requires": {
-        "cyclist": "~0.2.2",
+        "cyclist": "^1.0.1",
         "inherits": "^2.0.3",
         "readable-stream": "^2.1.5"
       }
@@ -11788,13 +11761,12 @@
       }
     },
     "parse-asn1": {
-      "version": "5.1.5",
-      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.5.tgz",
-      "integrity": "sha512-jkMYn1dcJqF6d5CpU689bq7w/b5ALS9ROVSpQDPrZsqqesUJii9qutvoT5ltGedNXMO2e16YUWIghG9KxaViTQ==",
+      "version": "5.1.6",
+      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
+      "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==",
       "requires": {
-        "asn1.js": "^4.0.0",
+        "asn1.js": "^5.2.0",
         "browserify-aes": "^1.0.0",
-        "create-hash": "^1.1.0",
         "evp_bytestokey": "^1.0.0",
         "pbkdf2": "^3.0.3",
         "safe-buffer": "^5.1.1"
@@ -11913,9 +11885,9 @@
       }
     },
     "pbkdf2": {
-      "version": "3.0.17",
-      "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.17.tgz",
-      "integrity": "sha512-U/il5MsrZp7mGg3mSQfn742na2T+1/vHDCG5/iTI3X9MKUuYUZVLQhyRsg06mCgDBTd57TxzgZt7P+fYfjRLtA==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz",
+      "integrity": "sha512-4Ejy1OPxi9f2tt1rRV7Go7zmfDQ+ZectEQz3VGUQhgq62HtIRPDyG/JtnwIxs6x3uNMwo2V7q1fMvKjb+Tnpqg==",
       "requires": {
         "create-hash": "^1.1.2",
         "create-hmac": "^1.1.4",
@@ -12218,6 +12190,8 @@
       "version": "7.3.1",
       "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
       "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==",
+      "dev": true,
+      "optional": true,
       "requires": {
         "asap": "~2.0.3"
       }
@@ -12238,13 +12212,23 @@
       }
     },
     "prop-types": {
-      "version": "15.6.0",
-      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.6.0.tgz",
-      "integrity": "sha1-zq8IMCL8RrSjX2nhPvda7Q1jmFY=",
+      "version": "15.7.2",
+      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz",
+      "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==",
       "requires": {
-        "fbjs": "^0.8.16",
-        "loose-envify": "^1.3.1",
-        "object-assign": "^4.1.1"
+        "loose-envify": "^1.4.0",
+        "object-assign": "^4.1.1",
+        "react-is": "^16.8.1"
+      },
+      "dependencies": {
+        "loose-envify": {
+          "version": "1.4.0",
+          "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
+          "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
+          "requires": {
+            "js-tokens": "^3.0.0 || ^4.0.0"
+          }
+        }
       }
     },
     "prop-types-exact": {
@@ -12330,16 +12314,16 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
     "pump": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
-      "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
+      "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
       "requires": {
         "end-of-stream": "^1.1.0",
         "once": "^1.3.1"
@@ -12353,6 +12337,17 @@
         "duplexify": "^3.6.0",
         "inherits": "^2.0.3",
         "pump": "^2.0.0"
+      },
+      "dependencies": {
+        "pump": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
+          "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
+          "requires": {
+            "end-of-stream": "^1.1.0",
+            "once": "^1.3.1"
+          }
+        }
       }
     },
     "punycode": {
@@ -13492,9 +13487,9 @@
       },
       "dependencies": {
         "lodash": {
-          "version": "4.17.15",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
-          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "version": "4.17.20",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+          "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
           "dev": true
         }
       }
@@ -13958,12 +13953,12 @@
       "dev": true
     },
     "selfsigned": {
-      "version": "1.10.7",
-      "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.7.tgz",
-      "integrity": "sha512-8M3wBCzeWIJnQfl43IKwOmC4H/RAp50S8DF60znzjW5GVqTcSe2vWclt7hmYVPkKPlHWOu5EaWOMZ2Y6W8ZXTA==",
+      "version": "1.10.8",
+      "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.8.tgz",
+      "integrity": "sha512-2P4PtieJeEwVgTU9QEcwIRDQ/mXJLX8/+I3ur+Pg16nS8oNbrGxEso9NyYWy8NAmXiNl4dlAp5MwoNeCWzON4w==",
       "dev": true,
       "requires": {
-        "node-forge": "0.9.0"
+        "node-forge": "^0.10.0"
       }
     },
     "semver": {
@@ -14001,10 +13996,13 @@
       }
     },
     "serialize-javascript": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-3.0.0.tgz",
-      "integrity": "sha512-skZcHYw2vEX4bw90nAr2iTTsz6x2SrHEnfxgKYmZlvJYBEZrvbKtobJWlQ20zczKb3bsHHXXTYt48zBA7ni9cw==",
-      "dev": true
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-3.1.0.tgz",
+      "integrity": "sha512-JIJT1DGiWmIKhzRsG91aS6Ze4sFUrYbltlkg2onR5OrnNM02Kl/hnY/T4FN2omvyeBbQmMJv+K4cPOpGzOTFBg==",
+      "dev": true,
+      "requires": {
+        "randombytes": "^2.1.0"
+      }
     },
     "serializerr": {
       "version": "1.0.3",
@@ -14644,9 +14642,9 @@
       }
     },
     "stream-shift": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz",
-      "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI="
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
+      "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ=="
     },
     "strict-uri-encode": {
       "version": "2.0.0",
@@ -15356,26 +15354,21 @@
       }
     },
     "terser-webpack-plugin": {
-      "version": "1.4.3",
-      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.3.tgz",
-      "integrity": "sha512-QMxecFz/gHQwteWwSo5nTc6UaICqN1bMedC5sMtUc7y3Ha3Q8y6ZO0iCR8pq4RJC8Hjf0FEPEHZqcMB/+DFCrA==",
+      "version": "1.4.5",
+      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz",
+      "integrity": "sha512-04Rfe496lN8EYruwi6oPQkG0vo8C+HT49X687FZnpPF0qMAIHONI6HEXYPKDOE8e5HjXTyKfqRd/agHtH0kOtw==",
       "requires": {
         "cacache": "^12.0.2",
         "find-cache-dir": "^2.1.0",
         "is-wsl": "^1.1.0",
         "schema-utils": "^1.0.0",
-        "serialize-javascript": "^2.1.2",
+        "serialize-javascript": "^4.0.0",
         "source-map": "^0.6.1",
         "terser": "^4.1.2",
         "webpack-sources": "^1.4.0",
         "worker-farm": "^1.7.0"
       },
       "dependencies": {
-        "bluebird": {
-          "version": "3.7.2",
-          "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
-          "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
-        },
         "cacache": {
           "version": "12.0.4",
           "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz",
@@ -15443,14 +15436,6 @@
             "path-exists": "^3.0.0"
           }
         },
-        "lru-cache": {
-          "version": "5.1.1",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
-          "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
-          "requires": {
-            "yallist": "^3.0.2"
-          }
-        },
         "make-dir": {
           "version": "2.1.0",
           "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
@@ -15460,23 +15445,6 @@
             "semver": "^5.6.0"
           }
         },
-        "mississippi": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz",
-          "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==",
-          "requires": {
-            "concat-stream": "^1.5.0",
-            "duplexify": "^3.4.2",
-            "end-of-stream": "^1.1.0",
-            "flush-write-stream": "^1.0.0",
-            "from2": "^2.1.0",
-            "parallel-transform": "^1.1.0",
-            "pump": "^3.0.0",
-            "pumpify": "^1.3.3",
-            "stream-each": "^1.1.0",
-            "through2": "^2.0.0"
-          }
-        },
         "p-limit": {
           "version": "2.3.0",
           "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
@@ -15511,24 +15479,13 @@
             "find-up": "^3.0.0"
           }
         },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        },
         "serialize-javascript": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-2.1.2.tgz",
-          "integrity": "sha512-rs9OggEUF0V4jUSecXazOYsLfu7OGK2qIn3c7IPBiffz32XniEp/TX9Xmc9LQfK2nQ2QKHvZ2oygKUGU0lG4jQ=="
-        },
-        "source-list-map": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz",
-          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw=="
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz",
+          "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==",
+          "requires": {
+            "randombytes": "^2.1.0"
+          }
         },
         "source-map": {
           "version": "0.6.1",
@@ -15542,15 +15499,6 @@
           "requires": {
             "figgy-pudding": "^3.5.1"
           }
-        },
-        "webpack-sources": {
-          "version": "1.4.3",
-          "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
-          "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
-          "requires": {
-            "source-list-map": "^2.0.0",
-            "source-map": "~0.6.1"
-          }
         }
       }
     },
@@ -15589,9 +15537,9 @@
       "dev": true
     },
     "timers-browserify": {
-      "version": "2.0.11",
-      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.11.tgz",
-      "integrity": "sha512-60aV6sgJ5YEbzUdn9c8kYGIqOubPoUdqQCul3SBAsRCZ40s6Y5cMcrW4dt3/k/EsbLVJNl9n6Vz3fTc+k2GeKQ==",
+      "version": "2.0.12",
+      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz",
+      "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==",
       "requires": {
         "setimmediate": "^1.0.4"
       }
@@ -15790,11 +15738,6 @@
       "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
       "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
     },
-    "ua-parser-js": {
-      "version": "0.7.20",
-      "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.20.tgz",
-      "integrity": "sha512-8OaIKfzL5cpx8eCMAhhvTlft8GYF8b2eQr6JkCyVdrgjcytyOmPCXrqXFcUnhonRpLlh5yxEZVohm6mzaowUOw=="
-    },
     "uglify-js": {
       "version": "3.9.3",
       "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.9.3.tgz",
@@ -16172,20 +16115,20 @@
       }
     },
     "watchpack": {
-      "version": "1.7.2",
-      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.2.tgz",
-      "integrity": "sha512-ymVbbQP40MFTp+cNMvpyBpBtygHnPzPkHqoIwRRj/0B8KhqQwV8LaKjtbaxF2lK4vl8zN9wCxS46IFCU5K4W0g==",
+      "version": "1.7.5",
+      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.5.tgz",
+      "integrity": "sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==",
       "requires": {
-        "chokidar": "^3.4.0",
+        "chokidar": "^3.4.1",
         "graceful-fs": "^4.1.2",
         "neo-async": "^2.5.0",
-        "watchpack-chokidar2": "^2.0.0"
+        "watchpack-chokidar2": "^2.0.1"
       }
     },
     "watchpack-chokidar2": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.0.tgz",
-      "integrity": "sha512-9TyfOyN/zLUbA288wZ8IsMZ+6cbzvsNyEzSBp6e/zkifi6xxbl8SmQ/CxQq32k8NNqrdVEVUVSEf56L4rQ/ZxA==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.1.tgz",
+      "integrity": "sha512-nCFfBIPKr5Sh61s4LPpy1Wtfi0HE8isJ3d2Yb5/Ppw2P2B/3eVSEBjKfN0fmHJSK14+31KwMKmcrzs2GM4P0Ww==",
       "optional": true,
       "requires": {
         "chokidar": "^2.1.8"
@@ -16447,9 +16390,9 @@
       "dev": true
     },
     "webpack": {
-      "version": "4.43.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.43.0.tgz",
-      "integrity": "sha512-GW1LjnPipFW2Y78OOab8NJlCflB7EFskMih2AHdvjbpKMeDJqEgSx24cXXXiPS65+WSwVyxtDsJH6jGX2czy+g==",
+      "version": "4.44.2",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.44.2.tgz",
+      "integrity": "sha512-6KJVGlCxYdISyurpQ0IPTklv+DULv05rs2hseIXer6D7KrUicRDLFb4IUM1S6LUAKypPM/nSiVSuv8jHu1m3/Q==",
       "requires": {
         "@webassemblyjs/ast": "1.9.0",
         "@webassemblyjs/helper-module-context": "1.9.0",
@@ -16459,7 +16402,7 @@
         "ajv": "^6.10.2",
         "ajv-keywords": "^3.4.1",
         "chrome-trace-event": "^1.0.2",
-        "enhanced-resolve": "^4.1.0",
+        "enhanced-resolve": "^4.3.0",
         "eslint-scope": "^4.0.3",
         "json-parse-better-errors": "^1.0.2",
         "loader-runner": "^2.4.0",
@@ -16472,14 +16415,14 @@
         "schema-utils": "^1.0.0",
         "tapable": "^1.1.3",
         "terser-webpack-plugin": "^1.4.3",
-        "watchpack": "^1.6.1",
+        "watchpack": "^1.7.4",
         "webpack-sources": "^1.4.1"
       },
       "dependencies": {
         "ajv": {
-          "version": "6.12.2",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz",
-          "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==",
+          "version": "6.12.6",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+          "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
           "requires": {
             "fast-deep-equal": "^3.1.1",
             "fast-json-stable-stringify": "^2.0.0",
@@ -16578,9 +16521,9 @@
           }
         },
         "fast-deep-equal": {
-          "version": "3.1.1",
-          "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz",
-          "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA=="
+          "version": "3.1.3",
+          "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+          "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
         },
         "is-accessor-descriptor": {
           "version": "0.1.6",
@@ -16661,29 +16604,10 @@
             "to-regex": "^3.0.2"
           }
         },
-        "source-list-map": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz",
-          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw=="
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        },
         "tapable": {
           "version": "1.1.3",
           "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz",
           "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA=="
-        },
-        "webpack-sources": {
-          "version": "1.4.3",
-          "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
-          "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
-          "requires": {
-            "source-list-map": "^2.0.0",
-            "source-map": "~0.6.1"
-          }
         }
       }
     },
@@ -17605,7 +17529,6 @@
       "version": "1.4.3",
       "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
       "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
-      "dev": true,
       "requires": {
         "source-list-map": "^2.0.0",
         "source-map": "~0.6.1"
@@ -17614,14 +17537,12 @@
         "source-list-map": {
           "version": "2.0.1",
           "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz",
-          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==",
-          "dev": true
+          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw=="
         },
         "source-map": {
           "version": "0.6.1",
           "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
         }
       }
     },
@@ -17660,11 +17581,6 @@
         }
       }
     },
-    "whatwg-fetch": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz",
-      "integrity": "sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q=="
-    },
     "whatwg-mimetype": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz",
@@ -17780,9 +17696,9 @@
       "dev": true
     },
     "xtend": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
-      "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
     },
     "y18n": {
       "version": "4.0.0",

buildscripts/gateway-tests.sh

@@ -1,69 +0,0 @@
-#!/bin/bash
-#
-# MinIO Cloud Storage, (C) 2019 MinIO, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -e
-set -E
-set -o pipefail
-
-function start_minio_server()
-{
-    MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 \
-                    minio --quiet --json server /data --address 127.0.0.1:24242 > server.log 2>&1 &
-    server_pid=$!
-    sleep 10
-
-    echo "$server_pid"
-}
-
-function start_minio_gateway_s3()
-{
-    MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 \
-                    minio --quiet --json gateway s3 http://127.0.0.1:24242 \
-                    --address 127.0.0.1:24240 > gateway.log 2>&1 &
-    gw_pid=$!
-    sleep 10
-
-    echo "$gw_pid"
-}
-
-function main()
-{
-    sr_pid="$(start_minio_server)"
-    gw_pid="$(start_minio_gateway_s3)"
-
-    SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \
-                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh \
-                   aws-sdk-go aws-sdk-java aws-sdk-php aws-sdk-ruby awscli \
-                   healthcheck mc minio-dotnet minio-js \
-                   minio-py s3cmd s3select security
-    rv=$?
-
-    kill "$sr_pid"
-    kill "$gw_pid"
-    sleep 3
-
-    if [ "$rv" -ne 0 ]; then
-        echo "=========== Gateway ==========="
-        cat "gateway.log"
-        echo "=========== Server ==========="
-        cat "server.log"
-    fi
-
-    rm -f gateway.log server.log
-}
-
-main "$@"

buildscripts/gen-ldflags.go

@@ -44,10 +44,21 @@ func releaseTag(version string) string {
 		relPrefix = prefix
 	}
 
+	relSuffix := ""
+	if hotfix := os.Getenv("MINIO_HOTFIX"); hotfix != "" {
+		relSuffix = hotfix
+	}
+
 	relTag := strings.Replace(version, " ", "-", -1)
 	relTag = strings.Replace(relTag, ":", "-", -1)
 	relTag = strings.Replace(relTag, ",", "", -1)
-	return relPrefix + "." + relTag
+	relTag = relPrefix + "." + relTag
+
+	if relSuffix != "" {
+		relTag += "." + relSuffix
+	}
+
+	return relTag
 }
 
 // commitID returns the abbreviated commit-id hash of the last commit.
@@ -68,5 +79,12 @@ func commitID() string {
 }
 
 func main() {
-	fmt.Println(genLDFlags(time.Now().UTC().Format(time.RFC3339)))
+	var version string
+	if len(os.Args) > 1 {
+		version = os.Args[1]
+	} else {
+		version = time.Now().UTC().Format(time.RFC3339)
+	}
+
+	fmt.Println(genLDFlags(version))
 }

buildscripts/race.sh

@@ -3,5 +3,5 @@
 set -e
 
 for d in $(go list ./... | grep -v browser); do
-    CGO_ENABLED=1 go test -v -race --timeout 50m "$d"
+    CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
 done

buildscripts/verify-build.sh

@@ -56,40 +56,41 @@ function start_minio_erasure()
 
 function start_minio_erasure_sets()
 {
-    "${MINIO[@]}" server "${WORK_DIR}/erasure-disk-sets{1...32}" >"$WORK_DIR/erasure-minio-sets.log" 2>&1 &
+    export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}"
+    "${MINIO[@]}" server > "$WORK_DIR/erasure-minio-sets.log" 2>&1 &
     sleep 15
 }
 
-function start_minio_zone_erasure_sets()
+function start_minio_pool_erasure_sets()
 {
-    export MINIO_ACCESS_KEY=$ACCESS_KEY
-    export MINIO_SECRET_KEY=$SECRET_KEY
-
-    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
+    export MINIO_ROOT_USER=$ACCESS_KEY
+    export MINIO_ROOT_PASSWORD=$SECRET_KEY
+    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
+    "${MINIO[@]}" server --address ":9000" > "$WORK_DIR/pool-minio-9000.log" 2>&1 &
+    "${MINIO[@]}" server --address ":9001" > "$WORK_DIR/pool-minio-9001.log" 2>&1 &
 
     sleep 40
 }
 
-function start_minio_zone_erasure_sets_ipv6()
+function start_minio_pool_erasure_sets_ipv6()
 {
-    export MINIO_ACCESS_KEY=$ACCESS_KEY
-    export MINIO_SECRET_KEY=$SECRET_KEY
-
-    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9001.log" 2>&1 &
+    export MINIO_ROOT_USER=$ACCESS_KEY
+    export MINIO_ROOT_PASSWORD=$SECRET_KEY
+    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets{5...8}"
+    "${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
+    "${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
 
     sleep 40
 }
 
 function start_minio_dist_erasure()
 {
-    export MINIO_ACCESS_KEY=$ACCESS_KEY
-    export MINIO_SECRET_KEY=$SECRET_KEY
-    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
-    "${MINIO[@]}" server --address=:9002 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9002.log" 2>&1 &
-    "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
+    export MINIO_ROOT_USER=$ACCESS_KEY
+    export MINIO_ROOT_PASSWORD=$SECRET_KEY
+    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
+    for i in $(seq 0 3); do
+        "${MINIO[@]}" server --address ":900${i}" > "$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
+    done
 
     sleep 40
 }
@@ -112,7 +113,8 @@ function run_test_fs()
     return "$rv"
 }
 
-function run_test_erasure_sets() {
+function run_test_erasure_sets()
+{
     start_minio_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
@@ -129,9 +131,9 @@ function run_test_erasure_sets() {
     return "$rv"
 }
 
-function run_test_zone_erasure_sets()
+function run_test_pool_erasure_sets()
 {
-    start_minio_zone_erasure_sets
+    start_minio_pool_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
@@ -142,20 +144,20 @@ function run_test_zone_erasure_sets()
     if [ "$rv" -ne 0 ]; then
         for i in $(seq 0 1); do
             echo "server$i log:"
-            cat "$WORK_DIR/zone-minio-900$i.log"
+            cat "$WORK_DIR/pool-minio-900$i.log"
         done
     fi
 
     for i in $(seq 0 1); do
-        rm -f "$WORK_DIR/zone-minio-900$i.log"
+        rm -f "$WORK_DIR/pool-minio-900$i.log"
     done
 
     return "$rv"
 }
 
-function run_test_zone_erasure_sets_ipv6()
+function run_test_pool_erasure_sets_ipv6()
 {
-    start_minio_zone_erasure_sets_ipv6
+    start_minio_pool_erasure_sets_ipv6
 
     export SERVER_ENDPOINT="[::1]:9000"
 
@@ -168,12 +170,12 @@ function run_test_zone_erasure_sets_ipv6()
     if [ "$rv" -ne 0 ]; then
         for i in $(seq 0 1); do
             echo "server$i log:"
-            cat "$WORK_DIR/zone-minio-ipv6-900$i.log"
+            cat "$WORK_DIR/pool-minio-ipv6-900$i.log"
         done
     fi
 
     for i in $(seq 0 1); do
-        rm -f "$WORK_DIR/zone-minio-ipv6-900$i.log"
+        rm -f "$WORK_DIR/pool-minio-ipv6-900$i.log"
     done
 
     return "$rv"
@@ -220,7 +222,7 @@ function run_test_dist_erasure()
 
     rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log"
 
-   return "$rv"
+    return "$rv"
 }
 
 function purge()
@@ -293,14 +295,14 @@ function main()
     fi
 
     echo "Testing in Distributed Eraure expanded setup"
-    if ! run_test_zone_erasure_sets; then
+    if ! run_test_pool_erasure_sets; then
         echo "FAILED"
         purge "$WORK_DIR"
         exit 1
     fi
 
     echo "Testing in Distributed Erasure expanded setup with ipv6"
-    if ! run_test_zone_erasure_sets_ipv6; then
+    if ! run_test_pool_erasure_sets_ipv6; then
         echo "FAILED"
         purge "$WORK_DIR"
         exit 1

buildscripts/verify-healing.sh

@@ -29,44 +29,52 @@ MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
 
 function start_minio_3_node() {
-    declare -a minio_pids
-    declare -a ARGS
-    export MINIO_ACCESS_KEY=minio
-    export MINIO_SECRET_KEY=minio123
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
     export MINIO_ERASURE_SET_DRIVE_COUNT=6
 
     start_port=$(shuf -i 10000-65000 -n 1)
+    args=""
     for i in $(seq 1 3); do
-        ARGS+=("http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/")
+        args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
     done
 
-    "${MINIO[@]}" --address ":$[$start_port+1]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
-    minio_pids[0]=$!
-    disown "${minio_pids[0]}"
+    "${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
+    disown $!
 
-    "${MINIO[@]}" --address ":$[$start_port+2]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
-    minio_pids[1]=$!
-    disown "${minio_pids[1]}"
+    "${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
+    disown $!
 
-    "${MINIO[@]}" --address ":$[$start_port+3]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
-    minio_pids[2]=$!
-    disown "${minio_pids[2]}"
+    "${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
+    disown $!
 
     sleep "$1"
-    for pid in "${minio_pids[@]}"; do
-        if ! kill "$pid"; then
-            for i in $(seq 1 3); do
-                echo "server$i log:"
-                cat "${WORK_DIR}/dist-minio-server$i.log"
-            done
-            echo "FAILED"
-            purge "$WORK_DIR"
-            exit 1
-        fi
+    if [ "$(pgrep -c minio)" -ne 3 ]; then
+        for i in $(seq 1 3); do
+            echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-server$i.log"
+        done
+        echo "FAILED"
+        purge "$WORK_DIR"
+        exit 1
+    fi
+    if ! pkill minio; then
+        for i in $(seq 1 3); do
+            echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-server$i.log"
+        done
+        echo "FAILED"
+        purge "$WORK_DIR"
+        exit 1
+    fi
+
+    sleep 1;
+    if pgrep minio; then
         # forcibly killing, to proceed further properly.
-        kill -9 "$pid"
-        sleep 1 # wait 1sec per pid
-    done
+        if ! pkill -9 minio; then
+            echo "no minio process running anymore, proceed."
+        fi
+    fi
 }
 
 

cmd/admin-bucket-handlers.go

@@ -20,13 +20,10 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
-	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/env"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 )
@@ -55,12 +52,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	// Turn off quota commands if data usage info is unavailable.
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminBucketQuotaDisabled), r.URL)
-		return
-	}
-
 	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -128,6 +119,12 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	defer logger.AuditLog(w, r, "SetBucketTarget", mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
+	update := r.URL.Query().Get("update") == "true"
+
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
 
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
@@ -135,10 +132,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	if !globalIsErasure {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
 
 	// Check if bucket exists.
 	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
@@ -163,23 +156,23 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	host, port, err := net.SplitHostPort(target.Endpoint)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
-		return
-	}
-	sameTarget, _ := isLocalHost(host, port, globalMinioPort)
+
+	sameTarget, _ := isLocalHost(target.URL().Hostname(), target.URL().Port(), globalMinioPort)
 	if sameTarget && bucket == target.TargetBucket {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL)
 		return
 	}
+
 	target.SourceBucket = bucket
-	target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
+	if !update {
+		target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
+	}
 	if target.Arn == "" {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target); err != nil {
+
+	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, update); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -187,14 +180,12 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
-
 	}
 	tgtBytes, err := json.Marshal(&targets)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-
 	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -218,7 +209,10 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	arnType := vars["type"]
-
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketTargetAction)
 	if objectAPI == nil {
@@ -255,16 +249,16 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 	arn := vars["arn"]
 
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	if !globalIsErasure {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
 
 	// Check if bucket exists.
 	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {

cmd/admin-handlers-config-kv.go

@@ -49,7 +49,7 @@ func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *htt
 	}
 
 	// Validate request signature.
-	cred, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
+	cred, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return cred, nil
@@ -130,8 +130,8 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+	dynamic, err := cfg.ReadConfig(bytes.NewReader(kvBytes))
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -158,6 +158,17 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		saveConfig(GlobalContext, objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 	}
 
+	// Apply dynamic values.
+	if err := applyDynamicConfig(GlobalContext, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	globalNotificationSys.SignalService(serviceReloadDynamic)
+
+	// If all values were dynamic, tell the client.
+	if dynamic {
+		w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue)
+	}
 	writeSuccessResponseHeadersOnly(w)
 }
 
@@ -266,7 +277,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+	if _, err = cfg.ReadConfig(bytes.NewReader(kvBytes)); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -378,7 +389,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	cfg := newServerConfig()
-	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+	if _, err = cfg.ReadConfig(bytes.NewReader(kvBytes)); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

cmd/admin-handlers-users.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"path"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
@@ -42,7 +43,7 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 	}
 
 	// Validate request signature.
-	cred, adminAPIErr = checkAdminRequestAuthType(ctx, r, action, "")
+	cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return nil, cred
@@ -129,13 +130,40 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 
 	defer logger.AuditLog(w, r, "GetUserInfo", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
-	if objectAPI == nil {
+	vars := mux.Vars(r)
+	name := vars["accessKey"]
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	name := vars["accessKey"]
+	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	accessKey := cred.AccessKey
+	if cred.ParentUser != "" {
+		accessKey = cred.ParentUser
+	}
+
+	implicitPerm := name == accessKey
+	if !implicitPerm {
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     accessKey,
+			Action:          iampolicy.GetUserAdminAction,
+			ConditionValues: getConditionValues(r, "", accessKey, claims),
+			IsOwner:         owner,
+			Claims:          claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	}
 
 	userInfo, err := globalIAMSys.GetUserInfo(name)
 	if err != nil {
@@ -304,7 +332,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	accessKey := vars["accessKey"]
 	status := vars["status"]
 
-	// Custom IAM policies not allowed for admin user.
+	// This API is not allowed to lookup accessKey user status
 	if accessKey == globalActiveCred.AccessKey {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
@@ -330,20 +358,53 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 
 	defer logger.AuditLog(w, r, "AddUser", mustGetClaimsFromToken(r))
 
-	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
-	if objectAPI == nil {
+	vars := mux.Vars(r)
+	accessKey := path.Clean(vars["accessKey"])
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	accessKey := vars["accessKey"]
+	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
 
-	// Custom IAM policies not allowed for admin user.
-	if accessKey == globalActiveCred.AccessKey {
+	// Not allowed to add a user with same access key as root credential
+	if owner && accessKey == cred.AccessKey {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
 		return
 	}
 
+	if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
+		// Incoming access key matches parent user then we should
+		// reject password change requests.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
+		return
+	}
+
+	implicitPerm := accessKey == cred.AccessKey
+	if !implicitPerm {
+		parentUser := cred.ParentUser
+		if parentUser == "" {
+			parentUser = cred.AccessKey
+		}
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     parentUser,
+			Action:          iampolicy.CreateUserAdminAction,
+			ConditionValues: getConditionValues(r, "", parentUser, claims),
+			IsOwner:         owner,
+			Claims:          claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	}
+
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
@@ -398,6 +459,12 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
+	// Disallow creating service accounts by root user.
+	if owner {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminAccountNotEligible), r.URL)
+		return
+	}
+
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
@@ -411,12 +478,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Disallow creating service accounts by root user.
-	if owner {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminAccountNotEligible), r.URL)
-		return
-	}
-
 	parentUser := cred.AccessKey
 	if cred.ParentUser != "" {
 		parentUser = cred.ParentUser
@@ -572,11 +633,11 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	writeSuccessNoContent(w)
 }
 
-// AccountUsageInfoHandler returns usage
-func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AccountUsageInfo")
+// AccountInfoHandler returns usage
+func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AccountInfo")
 
-	defer logger.AuditLog(w, r, "AccountUsageInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(w, r, "AccountInfo", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -645,8 +706,16 @@ func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http
 		accountName = cred.ParentUser
 	}
 
-	acctInfo := madmin.AccountUsageInfo{
+	policies, err := globalIAMSys.PolicyDBGet(accountName, false)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	acctInfo := madmin.AccountInfo{
 		AccountName: accountName,
+		Policy:      globalIAMSys.GetCombinedPolicy(policies...),
 	}
 
 	for _, bucket := range buckets {

cmd/admin-handlers.go

@@ -41,6 +41,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bandwidth"
 	"github.com/minio/minio/pkg/handlers"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -126,7 +127,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if lrTime.Sub(crTime) < 0 {
+	if lrTime.Sub(crTime) <= 0 {
 		updateStatus := madmin.ServerUpdateStatus{
 			CurrentVersion: Version,
 			UpdatedVersion: Version,
@@ -204,9 +205,10 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	}
 
 	var objectAPI ObjectLayer
-	if serviceSig == serviceRestart {
+	switch serviceSig {
+	case serviceRestart:
 		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
-	} else {
+	case serviceStop:
 		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
 	}
 	if objectAPI == nil {
@@ -290,7 +292,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	// ignores any errors here.
-	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
+	storageInfo, _ := objectAPI.StorageInfo(ctx)
 
 	// Collect any disk healing.
 	healing, _ := getAggregatedBackgroundHealState(ctx)
@@ -371,14 +373,12 @@ func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 		if peerLock == nil {
 			continue
 		}
-		for _, locks := range peerLock.Locks {
-			for k, v := range locks {
-				for _, lockReqInfo := range v {
-					if val, ok := entryMap[lockReqInfo.UID]; ok {
-						val.ServerList = append(val.ServerList, peerLock.Addr)
-					} else {
-						entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
-					}
+		for k, v := range peerLock.Locks {
+			for _, lockReqInfo := range v {
+				if val, ok := entryMap[lockReqInfo.UID]; ok {
+					val.ServerList = append(val.ServerList, peerLock.Addr)
+				} else {
+					entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
 				}
 			}
 		}
@@ -400,7 +400,7 @@ func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 // PeerLocks holds server information result of one node
 type PeerLocks struct {
 	Addr  string
-	Locks GetLocksResp
+	Locks map[string][]lockRequesterInfo
 }
 
 // TopLocksHandler Get list of locks in use
@@ -461,8 +461,15 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 
 	defer logger.AuditLog(w, r, "StartProfiling", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
-	if objectAPI == nil {
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
@@ -559,8 +566,15 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
 
 	defer logger.AuditLog(w, r, "DownloadProfiling", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
-	if objectAPI == nil {
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
@@ -794,7 +808,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	case hip.clientToken == "":
 		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), hip.hs, hip.forceStart)
 		go func() {
-			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
+			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh, objectAPI)
 			hr := healResp{respBytes, apiErr, errMsg}
 			respCh <- hr
 		}()
@@ -900,7 +914,7 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 	}
 
 	// Validate request signature.
-	cred, adminAPIErr = checkAdminRequestAuthType(ctx, r, action, "")
+	cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return nil, cred
@@ -1006,6 +1020,15 @@ func mustTrace(entry interface{}, trcAll, errOnly bool) bool {
 	if !ok {
 		return false
 	}
+
+	// Handle browser requests separately filter them and return.
+	if HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+"/upload") {
+		if errOnly {
+			return trcInfo.RespInfo.StatusCode >= http.StatusBadRequest
+		}
+		return true
+	}
+
 	trace := trcAll || !HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator)
 	if errOnly {
 		return trace && trcInfo.RespInfo.StatusCode >= http.StatusBadRequest
@@ -1023,7 +1046,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	trcErr := r.URL.Query().Get("err") == "true"
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.TraceAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -1035,7 +1058,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	traceCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalHTTPTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool {
 		return mustTrace(entry, trcAll, trcErr)
@@ -1103,7 +1126,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 	logCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
 
@@ -1235,26 +1258,26 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, resp)
 }
 
-// OBDInfoHandler - GET /minio/admin/v3/obdinfo
+// HealthInfoHandler - GET /minio/admin/v3/healthinfo
 // ----------
-// Get server on-board diagnostics
-func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "OBDInfo")
+// Get server health info
+func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "HealthInfo")
 
-	defer logger.AuditLog(w, r, "OBDInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(w, r, "HealthInfo", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.OBDInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
 	query := r.URL.Query()
-	obdInfo := madmin.OBDInfo{}
-	obdInfoCh := make(chan madmin.OBDInfo)
+	healthInfo := madmin.HealthInfo{}
+	healthInfoCh := make(chan madmin.HealthInfo)
 
 	enc := json.NewEncoder(w)
-	partialWrite := func(oinfo madmin.OBDInfo) {
-		obdInfoCh <- oinfo
+	partialWrite := func(oinfo madmin.HealthInfo) {
+		healthInfoCh <- oinfo
 	}
 
 	setCommonHeaders(w)
@@ -1267,8 +1290,8 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
 			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
 		encodedErrorResponse := encodeResponse(errorResponse)
-		obdInfo.Error = string(encodedErrorResponse)
-		logger.LogIf(ctx, enc.Encode(obdInfo))
+		healthInfo.Error = string(encodedErrorResponse)
+		logger.LogIf(ctx, enc.Encode(healthInfo))
 	}
 
 	deadline := 3600 * time.Second
@@ -1284,117 +1307,117 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
 	defer cancel()
 
-	nsLock := objectAPI.NewNSLock(ctx, minioMetaBucket, "obd-in-progress")
-	if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress")
+	if err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
 		errResp(err)
 		return
 	}
 	defer nsLock.Unlock()
 
 	go func() {
-		defer close(obdInfoCh)
+		defer close(healthInfoCh)
 
 		if cpu := query.Get("syscpu"); cpu == "true" {
-			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
+			cpuInfo := getLocalCPUInfo(deadlinedCtx, r)
 
-			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
-			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, globalNotificationSys.CPUOBDInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
+			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, cpuInfo)
+			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, globalNotificationSys.CPUInfo(deadlinedCtx)...)
+			partialWrite(healthInfo)
 		}
 
 		if diskHw := query.Get("sysdiskhw"); diskHw == "true" {
-			diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r)
+			diskHwInfo := getLocalDiskHwInfo(deadlinedCtx, r)
 
-			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
-			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwOBDInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
+			healthInfo.Sys.DiskHwInfo = append(healthInfo.Sys.DiskHwInfo, diskHwInfo)
+			healthInfo.Sys.DiskHwInfo = append(healthInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwInfo(deadlinedCtx)...)
+			partialWrite(healthInfo)
 		}
 
 		if osInfo := query.Get("sysosinfo"); osInfo == "true" {
-			osInfo := getLocalOsInfoOBD(deadlinedCtx, r)
+			osInfo := getLocalOsInfo(deadlinedCtx, r)
 
-			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
-			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, globalNotificationSys.OsOBDInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
+			healthInfo.Sys.OsInfo = append(healthInfo.Sys.OsInfo, osInfo)
+			healthInfo.Sys.OsInfo = append(healthInfo.Sys.OsInfo, globalNotificationSys.OsInfo(deadlinedCtx)...)
+			partialWrite(healthInfo)
 		}
 
 		if mem := query.Get("sysmem"); mem == "true" {
-			memInfo := getLocalMemOBD(deadlinedCtx, r)
+			memInfo := getLocalMemInfo(deadlinedCtx, r)
 
-			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
-			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, globalNotificationSys.MemOBDInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
+			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, memInfo)
+			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, globalNotificationSys.MemInfo(deadlinedCtx)...)
+			partialWrite(healthInfo)
 		}
 
 		if proc := query.Get("sysprocess"); proc == "true" {
-			procInfo := getLocalProcOBD(deadlinedCtx, r)
+			procInfo := getLocalProcInfo(deadlinedCtx, r)
 
-			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
-			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, globalNotificationSys.ProcOBDInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
+			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, procInfo)
+			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, globalNotificationSys.ProcInfo(deadlinedCtx)...)
+			partialWrite(healthInfo)
 		}
 
 		if config := query.Get("minioconfig"); config == "true" {
 			cfg, err := readServerConfig(ctx, objectAPI)
 			logger.LogIf(ctx, err)
-			obdInfo.Minio.Config = cfg
-			partialWrite(obdInfo)
+			healthInfo.Minio.Config = cfg
+			partialWrite(healthInfo)
 		}
 
 		if drive := query.Get("perfdrive"); drive == "true" {
-			// Get drive obd details from local server's drive(s)
-			driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
-			driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
+			// Get drive perf details from local server's drive(s)
+			drivePerfSerial := getLocalDrives(deadlinedCtx, false, globalEndpoints, r)
+			drivePerfParallel := getLocalDrives(deadlinedCtx, true, globalEndpoints, r)
 
 			errStr := ""
-			if driveOBDSerial.Error != "" {
-				errStr = "serial: " + driveOBDSerial.Error
+			if drivePerfSerial.Error != "" {
+				errStr = "serial: " + drivePerfSerial.Error
 			}
-			if driveOBDParallel.Error != "" {
-				errStr = errStr + " parallel: " + driveOBDParallel.Error
+			if drivePerfParallel.Error != "" {
+				errStr = errStr + " parallel: " + drivePerfParallel.Error
 			}
 
-			driveOBD := madmin.ServerDrivesOBDInfo{
-				Addr:     driveOBDSerial.Addr,
-				Serial:   driveOBDSerial.Serial,
-				Parallel: driveOBDParallel.Parallel,
+			driveInfo := madmin.ServerDrivesInfo{
+				Addr:     drivePerfSerial.Addr,
+				Serial:   drivePerfSerial.Serial,
+				Parallel: drivePerfParallel.Parallel,
 				Error:    errStr,
 			}
-			obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBD)
-			partialWrite(obdInfo)
+			healthInfo.Perf.DriveInfo = append(healthInfo.Perf.DriveInfo, driveInfo)
+			partialWrite(healthInfo)
 
-			// Notify all other MinIO peers to report drive obd numbers
-			driveOBDs := globalNotificationSys.DriveOBDInfoChan(deadlinedCtx)
-			for obd := range driveOBDs {
-				obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, obd)
-				partialWrite(obdInfo)
+			// Notify all other MinIO peers to report drive perf numbers
+			driveInfos := globalNotificationSys.DrivePerfInfoChan(deadlinedCtx)
+			for obd := range driveInfos {
+				healthInfo.Perf.DriveInfo = append(healthInfo.Perf.DriveInfo, obd)
+				partialWrite(healthInfo)
 			}
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 		}
 
 		if net := query.Get("perfnet"); net == "true" && globalIsDistErasure {
-			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
-			partialWrite(obdInfo)
+			healthInfo.Perf.Net = append(healthInfo.Perf.Net, globalNotificationSys.NetInfo(deadlinedCtx))
+			partialWrite(healthInfo)
 
-			netOBDs := globalNotificationSys.DispatchNetOBDChan(deadlinedCtx)
-			for obd := range netOBDs {
-				obdInfo.Perf.Net = append(obdInfo.Perf.Net, obd)
-				partialWrite(obdInfo)
+			netInfos := globalNotificationSys.DispatchNetPerfChan(deadlinedCtx)
+			for netInfo := range netInfos {
+				healthInfo.Perf.Net = append(healthInfo.Perf.Net, netInfo)
+				partialWrite(healthInfo)
 			}
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 
-			obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
-			partialWrite(obdInfo)
+			healthInfo.Perf.NetParallel = globalNotificationSys.NetPerfParallelInfo(deadlinedCtx)
+			partialWrite(healthInfo)
 		}
 
 	}()
 
-	ticker := time.NewTicker(30 * time.Second)
+	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
 
 	for {
 		select {
-		case oinfo, ok := <-obdInfoCh:
+		case oinfo, ok := <-healthInfoCh:
 			if !ok {
 				return
 			}
@@ -1419,23 +1442,51 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "BandwidthMonitor")
 
+	defer logger.AuditLog(w, r, "BandwidthMonitor", mustGetClaimsFromToken(r))
+
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.BandwidthMonitorAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.BandwidthMonitorAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
 	}
 
 	setEventStreamHeaders(w)
+	reportCh := make(chan bandwidth.Report, 1)
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
 	bucketsRequestedString := r.URL.Query().Get("buckets")
 	bucketsRequested := strings.Split(bucketsRequestedString, ",")
-	consolidatedReport := globalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...)
-	enc := json.NewEncoder(w)
-	err := enc.Encode(consolidatedReport)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+	go func() {
+		for {
+			reportCh <- globalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...)
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				time.Sleep(2 * time.Second)
+			}
+		}
+	}()
+	for {
+		select {
+		case report := <-reportCh:
+			enc := json.NewEncoder(w)
+			err := enc.Encode(report)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-keepAliveTicker.C:
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-ctx.Done():
+			return
+		}
 	}
-	w.(http.Flusher).Flush()
 }
 
 // ServerInfoHandler - GET /minio/admin/v3/info
@@ -1446,22 +1497,13 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	defer logger.AuditLog(w, r, "ServerInfo", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
-	if objectAPI == nil {
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
 	}
 
-	buckets := madmin.Buckets{}
-	objects := madmin.Objects{}
-	usage := madmin.Usage{}
-
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err == nil {
-		buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
-		objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
-		usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
-	}
-
 	vault := fetchVaultStatus()
 
 	ldap := madmin.LDAP{}
@@ -1483,31 +1525,54 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	// Get the notification target info
 	notifyTarget := fetchLambdaInfo()
 
-	// Fetching the Storage information, ignore any errors.
-	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
-
-	var backend interface{}
-	if storageInfo.Backend.Type == BackendType(madmin.Erasure) {
-		backend = madmin.ErasureBackend{
-			Type:             madmin.ErasureType,
-			OnlineDisks:      storageInfo.Backend.OnlineDisks.Sum(),
-			OfflineDisks:     storageInfo.Backend.OfflineDisks.Sum(),
-			StandardSCData:   storageInfo.Backend.StandardSCData,
-			StandardSCParity: storageInfo.Backend.StandardSCParity,
-			RRSCData:         storageInfo.Backend.RRSCData,
-			RRSCParity:       storageInfo.Backend.RRSCParity,
-		}
-	} else {
-		backend = madmin.FSBackend{
-			Type: madmin.FsType,
-		}
-	}
-
-	mode := "online"
 	server := getLocalServerProperty(globalEndpoints, r)
 	servers := globalNotificationSys.ServerInfo()
 	servers = append(servers, server)
 
+	var backend interface{}
+	mode := madmin.ObjectLayerInitializing
+
+	buckets := madmin.Buckets{}
+	objects := madmin.Objects{}
+	usage := madmin.Usage{}
+
+	objectAPI := newObjectLayerFn()
+	if objectAPI != nil {
+		mode = madmin.ObjectLayerOnline
+		// Load data usage
+		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+		if err == nil {
+			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
+			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
+			usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
+		}
+
+		// Fetching the backend information
+		backendInfo := objectAPI.BackendInfo()
+		if backendInfo.Type == BackendType(madmin.Erasure) {
+			// Calculate the number of online/offline disks of all nodes
+			var allDisks []madmin.Disk
+			for _, s := range servers {
+				allDisks = append(allDisks, s.Disks...)
+			}
+			onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
+
+			backend = madmin.ErasureBackend{
+				Type:             madmin.ErasureType,
+				OnlineDisks:      onlineDisks.Sum(),
+				OfflineDisks:     offlineDisks.Sum(),
+				StandardSCData:   backendInfo.StandardSCData,
+				StandardSCParity: backendInfo.StandardSCParity,
+				RRSCData:         backendInfo.RRSCData,
+				RRSCParity:       backendInfo.RRSCParity,
+			}
+		} else {
+			backend = madmin.FSBackend{
+				Type: madmin.FsType,
+			}
+		}
+	}
+
 	domain := globalDomainNames
 	services := madmin.Services{
 		Vault:         vault,
@@ -1517,25 +1582,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		Notifications: notifyTarget,
 	}
 
-	// find all disks which belong to each respective endpoints
-	for i := range servers {
-		for _, disk := range storageInfo.Disks {
-			if strings.Contains(disk.Endpoint, servers[i].Endpoint) {
-				servers[i].Disks = append(servers[i].Disks, disk)
-			}
-		}
-	}
-
-	// add all the disks local to this server.
-	for _, disk := range storageInfo.Disks {
-		if disk.DrivePath == "" && disk.Endpoint == "" {
-			continue
-		}
-		if disk.Endpoint == disk.DrivePath {
-			servers[len(servers)-1].Disks = append(servers[len(servers)-1].Disks, disk)
-		}
-	}
-
 	infoMsg := madmin.InfoMessage{
 		Mode:         mode,
 		Domain:       domain,

cmd/admin-handlers_test.go

@@ -105,8 +105,8 @@ func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error)
 	}
 
 	globalPolicySys = NewPolicySys()
-	objLayer := &erasureServerSets{serverSets: make([]*erasureSets, 1)}
-	objLayer.serverSets[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
+	objLayer := &erasureServerPools{serverPools: make([]*erasureSets, 1)}
+	objLayer.serverPools[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -364,7 +364,7 @@ func TestExtractHealInitParams(t *testing.T) {
 	// Test all combinations!
 	for pIdx, parms := range qParmsArr {
 		for vIdx, vars := range varsArr {
-			_, err := extractHealInitParams(vars, parms, bytes.NewBuffer([]byte(body)))
+			_, err := extractHealInitParams(vars, parms, bytes.NewReader([]byte(body)))
 			isErrCase := false
 			if pIdx < 4 || vIdx < 1 {
 				isErrCase = true

cmd/admin-heal-ops.go

@@ -21,6 +21,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -93,15 +95,15 @@ type allHealState struct {
 }
 
 // newHealState - initialize global heal state management
-func newHealState() *allHealState {
-	healState := &allHealState{
+func newHealState(cleanup bool) *allHealState {
+	hstate := &allHealState{
 		healSeqMap:     make(map[string]*healSequence),
 		healLocalDisks: map[Endpoint]struct{}{},
 	}
-
-	go healState.periodicHealSeqsClean(GlobalContext)
-
-	return healState
+	if cleanup {
+		go hstate.periodicHealSeqsClean(GlobalContext)
+	}
+	return hstate
 }
 
 func (ahs *allHealState) healDriveCount() int {
@@ -143,9 +145,13 @@ func (ahs *allHealState) pushHealLocalDisks(healLocalDisks ...Endpoint) {
 func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
+	periodicTimer := time.NewTimer(time.Minute * 5)
+	defer periodicTimer.Stop()
+
 	for {
 		select {
-		case <-time.After(time.Minute * 5):
+		case <-periodicTimer.C:
+			periodicTimer.Reset(time.Minute * 5)
 			now := UTCNow()
 			ahs.Lock()
 			for path, h := range ahs.healSeqMap {
@@ -228,7 +234,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 // `keepHealSeqStateDuration`. This function also launches a
 // background routine to clean up heal results after the
 // aforementioned duration.
-func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
+func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLayer) (
 	respBytes []byte, apiErr APIError, errMsg string) {
 
 	if h.forceStarted {
@@ -265,7 +271,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	ahs.healSeqMap[hpath] = h
 
 	// Launch top-level background heal go-routine
-	go h.healSequenceStart()
+	go h.healSequenceStart(objAPI)
 
 	clientToken := h.clientToken
 	if globalIsDistErasure {
@@ -609,7 +615,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 // routine for completion, and (2) listens for external stop
 // signals. When either event happens, it sets the finish status for
 // the heal-sequence.
-func (h *healSequence) healSequenceStart() {
+func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 	// Set status as running
 	h.mutex.Lock()
 	h.currentStatus.Summary = healRunningStatus
@@ -617,7 +623,7 @@ func (h *healSequence) healSequenceStart() {
 	h.mutex.Unlock()
 
 	if h.sourceCh == nil {
-		go h.traverseAndHeal()
+		go h.traverseAndHeal(objAPI)
 	} else {
 		go h.healFromSourceCh()
 	}
@@ -657,6 +663,10 @@ func (h *healSequence) healSequenceStart() {
 }
 
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
+	globalHealConfigMu.Lock()
+	opts := globalHealConfig
+	globalHealConfigMu.Unlock()
+
 	// Send heal request
 	task := healTask{
 		bucket:     source.bucket,
@@ -668,6 +678,12 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	if source.opts != nil {
 		task.opts = *source.opts
 	}
+	if opts.Bitrot {
+		task.opts.ScanMode = madmin.HealDeepScan
+	}
+
+	// Wait and proceed if there are active requests
+	waitForLowHTTPReq(opts.IOCount, opts.Sleep)
 
 	h.mutex.Lock()
 	h.scannedItemsMap[healType]++
@@ -766,28 +782,28 @@ func (h *healSequence) healFromSourceCh() {
 	h.healItemsFromSourceCh()
 }
 
-func (h *healSequence) healDiskMeta() error {
-	// Start with format healing
-	if err := h.healDiskFormat(); err != nil {
-		return err
+func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
+	// Try to pro-actively heal backend-encrypted file.
+	if err := h.queueHealTask(healSource{
+		bucket: minioMetaBucket,
+		object: backendEncryptedFile,
+	}, madmin.HealItemBucketMetadata); err != nil {
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return err
+		}
 	}
 
 	// Start healing the config prefix.
-	if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
-		return err
-	}
-
-	// Start healing the bucket config prefix.
-	return h.healMinioSysMeta(bucketConfigPrefix)()
+	return h.healMinioSysMeta(objAPI, minioConfigPrefix)()
 }
 
-func (h *healSequence) healItems(bucketsOnly bool) error {
-	if err := h.healDiskMeta(); err != nil {
+func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
+	if err := h.healDiskMeta(objAPI); err != nil {
 		return err
 	}
 
 	// Heal buckets and objects
-	return h.healBuckets(bucketsOnly)
+	return h.healBuckets(objAPI, bucketsOnly)
 }
 
 // traverseAndHeal - traverses on-disk data and performs healing
@@ -797,30 +813,29 @@ func (h *healSequence) healItems(bucketsOnly bool) error {
 // quit signal is received, this routine cannot quit immediately and
 // has to wait until a safe point is reached, such as between scanning
 // two objects.
-func (h *healSequence) traverseAndHeal() {
+func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
 	bucketsOnly := false // Heals buckets and objects also.
-	h.traverseAndHealDoneCh <- h.healItems(bucketsOnly)
+	h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
 	close(h.traverseAndHealDoneCh)
 }
 
 // healMinioSysMeta - heals all files under a given meta prefix, returns a function
 // which in-turn heals the respective meta directory path and any files in int.
-func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
+func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) func() error {
 	return func() error {
-		// Get current object layer instance.
-		objectAPI := newObjectLayerFn()
-		if objectAPI == nil {
-			return errServerNotInitialized
-		}
-
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
+		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
 
+			// Skip metacache entries healing
+			if strings.HasPrefix(object, "buckets/.minio.sys/.metacache/") {
+				return nil
+			}
+
 			err := h.queueHealTask(healSource{
 				bucket:    bucket,
 				object:    object,
@@ -843,39 +858,32 @@ func (h *healSequence) healDiskFormat() error {
 		return errHealStopSignalled
 	}
 
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	return h.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
-func (h *healSequence) healBuckets(bucketsOnly bool) error {
+func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
 
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
-		return h.healBucket(h.bucket, bucketsOnly)
+		return h.healBucket(objAPI, h.bucket, bucketsOnly)
 	}
 
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
-	buckets, err := objectAPI.ListBucketsHeal(h.ctx)
+	buckets, err := objAPI.ListBuckets(h.ctx)
 	if err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
 
+	// Heal latest buckets first.
+	sort.Slice(buckets, func(i, j int) bool {
+		return buckets[i].Created.After(buckets[j].Created)
+	})
+
 	for _, bucket := range buckets {
-		if err = h.healBucket(bucket.Name, bucketsOnly); err != nil {
+		if err = h.healBucket(objAPI, bucket.Name, bucketsOnly); err != nil {
 			return err
 		}
 	}
@@ -884,13 +892,7 @@ func (h *healSequence) healBuckets(bucketsOnly bool) error {
 }
 
 // healBucket - traverses and heals given bucket
-func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
+func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly bool) error {
 	if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil {
 		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 			return err
@@ -905,7 +907,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		if h.object != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			oi, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
+			oi, err := objAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
 			if err == nil {
 				if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
@@ -919,7 +921,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		return nil
 	}
 
-	if err := objectAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
+	if err := objAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
 		// Object might have been deleted, by the time heal
 		// was attempted we ignore this object an move on.
 		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
@@ -931,12 +933,6 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 
 // healObject - heal the given object and record result
 func (h *healSequence) healObject(bucket, object, versionID string) error {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}

cmd/admin-router.go

@@ -20,8 +20,6 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -116,7 +114,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 			// Add user IAM
 
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountUsageInfoHandler))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(httpTraceAll(adminAPI.AccountInfoHandler))
 
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
 
@@ -172,27 +170,25 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		}
 
 		if globalIsDistErasure || globalIsErasure {
-			// Quota operations
-			if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
-				// GetBucketQuotaConfig
-				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
-					httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
-				// PutBucketQuotaConfig
-				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
-					httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+			// GetBucketQuotaConfig
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
+				httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+			// PutBucketQuotaConfig
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
+				httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
 
-				// Bucket replication operations
-				// GetBucketTargetHandler
-				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
-					httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
-				// SetRemoteTargetHandler
-				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
-					httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
-				// SetRemoteTargetHandler
-				adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
-					httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
-			}
+			// Bucket replication operations
+			// GetBucketTargetHandler
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
+				httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+			// SetRemoteTargetHandler
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
+				httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
+			// RemoveRemoteTargetHandler
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
+				httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 		}
+
 		// -- Top APIs --
 		// Top locks
 		if globalIsDistErasure {
@@ -211,9 +207,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
 
 		if !globalIsGateway {
-			// -- OBD API --
+			// Keep obdinfo for backward compatibility with mc
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
-				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler))
+				HandlerFunc(httpTraceHdrs(adminAPI.HealthInfoHandler))
+			// -- Health API --
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
+				HandlerFunc(httpTraceHdrs(adminAPI.HealthInfoHandler))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
 				HandlerFunc(httpTraceHdrs(adminAPI.BandwidthMonitorHandler))
 		}
@@ -221,5 +220,5 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 	// If none of the routes match add default error handler routes
 	adminRouter.NotFoundHandler = httpTraceAll(errorResponseHandler)
-	adminRouter.MethodNotAllowedHandler = httpTraceAll(errorResponseHandler)
+	adminRouter.MethodNotAllowedHandler = httpTraceAll(methodNotAllowedHandler("Admin"))
 }

cmd/admin-server-info.go

@@ -24,13 +24,14 @@ import (
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
 // local endpoints from given list of endpoints
-func getLocalServerProperty(endpointServerSets EndpointServerSets, r *http.Request) madmin.ServerProperties {
+func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
+	var localEndpoints Endpoints
 	addr := r.Host
 	if globalIsDistErasure {
-		addr = GetLocalPeer(endpointServerSets)
+		addr = GetLocalPeer(endpointServerPools)
 	}
 	network := make(map[string]string)
-	for _, ep := range endpointServerSets {
+	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
 			nodeName := endpoint.Host
 			if nodeName == "" {
@@ -39,11 +40,12 @@ func getLocalServerProperty(endpointServerSets EndpointServerSets, r *http.Reque
 			if endpoint.IsLocal {
 				// Only proceed for local endpoints
 				network[nodeName] = "online"
+				localEndpoints = append(localEndpoints, endpoint)
 				continue
 			}
 			_, present := network[nodeName]
 			if !present {
-				if err := IsServerResolvable(endpoint); err == nil {
+				if err := isServerResolvable(endpoint); err == nil {
 					network[nodeName] = "online"
 				} else {
 					network[nodeName] = "offline"
@@ -52,6 +54,11 @@ func getLocalServerProperty(endpointServerSets EndpointServerSets, r *http.Reque
 		}
 	}
 
+	localDisks, _ := initStorageDisksWithErrors(localEndpoints)
+	defer closeStorageDisks(localDisks)
+
+	storageInfo, _ := getStorageInfo(localDisks, localEndpoints.GetAllStrings())
+
 	return madmin.ServerProperties{
 		State:    "ok",
 		Endpoint: addr,
@@ -59,5 +66,6 @@ func getLocalServerProperty(endpointServerSets EndpointServerSets, r *http.Reque
 		Version:  Version,
 		CommitID: CommitID,
 		Network:  network,
+		Disks:    storageInfo.Disks,
 	}
 }

cmd/api-datatypes.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"encoding/xml"
+	"time"
 )
 
 // DeletedObject objects deleted
@@ -26,12 +27,44 @@ type DeletedObject struct {
 	DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"`
 	ObjectName            string `xml:"Key,omitempty"`
 	VersionID             string `xml:"VersionId,omitempty"`
+
+	// MinIO extensions to support delete marker replication
+	// Replication status of DeleteMarker
+	DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus,omitempty"`
+	// MTime of DeleteMarker on source that needs to be propagated to replica
+	DeleteMarkerMTime DeleteMarkerMTime `xml:"DeleteMarkerMTime,omitempty"`
+	// Status of versioned delete (of object or DeleteMarker)
+	VersionPurgeStatus VersionPurgeStatusType `xml:"VersionPurgeStatus,omitempty"`
+	// PurgeTransitioned is nonempty if object is in transition tier
+	PurgeTransitioned string `xml:"PurgeTransitioned,omitempty"`
+}
+
+// DeleteMarkerMTime is an embedded type containing time.Time for XML marshal
+type DeleteMarkerMTime struct {
+	time.Time
+}
+
+// MarshalXML encodes expiration date if it is non-zero and encodes
+// empty string otherwise
+func (t DeleteMarkerMTime) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if t.Time.IsZero() {
+		return nil
+	}
+	return e.EncodeElement(t.Time.Format(time.RFC3339), startElement)
 }
 
 // ObjectToDelete carries key name for the object to delete.
 type ObjectToDelete struct {
 	ObjectName string `xml:"Key"`
 	VersionID  string `xml:"VersionId"`
+	// Replication status of DeleteMarker
+	DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus"`
+	// Status of versioned delete (of object or DeleteMarker)
+	VersionPurgeStatus VersionPurgeStatusType `xml:"VersionPurgeStatus"`
+	// Version ID of delete marker
+	DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId"`
+	// PurgeTransitioned is nonempty if object is in transition tier
+	PurgeTransitioned string `xml:"PurgeTransitioned"`
 }
 
 // createBucketConfiguration container for bucket configuration request from client.

cmd/api-errors.go

@@ -120,9 +120,10 @@ const (
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
-	ErrBucketReplicationDisabledError
+	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
 	ErrNoSuchUpload
+	ErrInvalidVersionID
 	ErrNoSuchVersion
 	ErrNotImplemented
 	ErrPreconditionFailed
@@ -233,6 +234,7 @@ const (
 	ErrInvalidResourceName
 	ErrServerNotInitialized
 	ErrOperationTimedOut
+	ErrClientDisconnected
 	ErrOperationMaxedOut
 	ErrInvalidRequest
 	// MinIO storage class error codes
@@ -260,7 +262,6 @@ const (
 	// Bucket Quota error codes
 	ErrAdminBucketQuotaExceeded
 	ErrAdminNoSuchQuotaConfiguration
-	ErrAdminBucketQuotaDisabled
 
 	ErrHealNotImplemented
 	ErrHealNoSuchProcess
@@ -361,6 +362,7 @@ const (
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
 	ErrAdminAccountNotEligible
+	ErrAccountNotEligible
 	ErrServiceAccountNotFound
 	ErrPostPolicyConditionInvalidFormat
 )
@@ -555,11 +557,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	ErrNoSuchVersion: {
+	ErrInvalidVersionID: {
 		Code:           "InvalidArgument",
 		Description:    "Invalid version id specified",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrNoSuchVersion: {
+		Code:           "NoSuchVersion",
+		Description:    "The specified version does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNotImplemented: {
 		Code:           "NotImplemented",
 		Description:    "A header you provided implies functionality that is not implemented",
@@ -827,7 +834,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrReplicationRemoteConnectionError: {
 		Code:           "XMinioAdminReplicationRemoteConnectionError",
-		Description:    "Remote service endpoint or target bucket not available",
+		Description:    "Remote service connection error - please check remote service credentials and target bucket",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrBucketRemoteIdenticalToSource: {
@@ -880,11 +887,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Versioning must be 'Enabled' on the bucket to add a replication target",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrBucketReplicationDisabledError: {
-		Code:           "XMinioAdminBucketReplicationDisabled",
-		Description:    "Replication specified but disk usage crawl is disabled on MinIO server",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
 		Description:    "The specified object does not have a ObjectLock configuration",
@@ -915,6 +917,11 @@ var errorCodes = errorCodeMap{
 		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectRestoreAlreadyInProgress: {
+		Code:           "RestoreAlreadyInProgress",
+		Description:    "Object restore is already in progress",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	/// Bucket notification related errors.
 	ErrEventNotification: {
 		Code:           "InvalidArgument",
@@ -1201,11 +1208,6 @@ var errorCodes = errorCodeMap{
 		Description:    "The quota configuration does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	ErrAdminBucketQuotaDisabled: {
-		Code:           "XMinioAdminBucketQuotaDisabled",
-		Description:    "Quota specified but disk usage crawl is disabled on MinIO server",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrInsecureClientRequest: {
 		Code:           "XMinioInsecureClientRequest",
 		Description:    "Cannot respond to plain-text request from TLS-encrypted server",
@@ -1216,6 +1218,11 @@ var errorCodes = errorCodeMap{
 		Description:    "A timeout occurred while trying to lock a resource, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrClientDisconnected: {
+		Code:           "ClientDisconnected",
+		Description:    "Client disconnected before response was ready",
+		HTTPStatusCode: 499, // No official code, use nginx value.
+	},
 	ErrOperationMaxedOut: {
 		Code:           "SlowDown",
 		Description:    "A timeout exceeded while waiting to proceed with the request, please reduce your request rate",
@@ -1714,12 +1721,17 @@ var errorCodes = errorCodeMap{
 	ErrAddUserInvalidArgument: {
 		Code:           "XMinioInvalidIAMCredentials",
 		Description:    "User is not allowed to be same as admin access key",
-		HTTPStatusCode: http.StatusConflict,
+		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrAdminAccountNotEligible: {
 		Code:           "XMinioInvalidIAMCredentials",
 		Description:    "The administrator key is not eligible for this operation",
-		HTTPStatusCode: http.StatusConflict,
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAccountNotEligible: {
+		Code:           "XMinioInvalidIAMCredentials",
+		Description:    "The account key is not eligible for this operation",
+		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrServiceAccountNotFound: {
 		Code:           "XMinioInvalidIAMCredentials",
@@ -1742,6 +1754,16 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
+	// Only return ErrClientDisconnected if the provided context is actually canceled.
+	// This way downstream context.Canceled will still report ErrOperationTimedOut
+	select {
+	case <-ctx.Done():
+		if ctx.Err() == context.Canceled {
+			return ErrClientDisconnected
+		}
+	default:
+	}
+
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument
@@ -1858,6 +1880,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrNoSuchKey
 	case MethodNotAllowed:
 		apiErr = ErrMethodNotAllowed
+	case InvalidVersionID:
+		apiErr = ErrInvalidVersionID
 	case VersionNotFound:
 		apiErr = ErrNoSuchVersion
 	case ObjectAlreadyExists:

cmd/api-headers.go

@@ -47,7 +47,9 @@ func setEventStreamHeaders(w http.ResponseWriter) {
 
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
-	w.Header().Set(xhttp.ServerInfo, "MinIO/"+ReleaseTag)
+	// Set the "Server" http header.
+	w.Header().Set(xhttp.ServerInfo, "MinIO")
+
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
 	if region := globalServerRegion; region != "" {
@@ -115,10 +117,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	// Set tag count if object has tags
-	tags, _ := url.ParseQuery(objInfo.UserTags)
-	tagCount := len(tags)
-	if tagCount > 0 {
-		w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tagCount)}
+	if len(objInfo.UserTags) > 0 {
+		tags, _ := url.ParseQuery(objInfo.UserTags)
+		if len(tags) > 0 {
+			w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(len(tags))}
+		}
 	}
 
 	// Set all other user defined metadata.
@@ -154,19 +157,13 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	if opts.PartNumber > 0 {
-		var start, end int64
-		for i := 0; i < len(objInfo.Parts) && i < opts.PartNumber; i++ {
-			start = end
-			end = start + objInfo.Parts[i].ActualSize - 1
-		}
-		rs = &HTTPRangeSpec{Start: start, End: end}
-		rangeLen = end - start + 1
-	} else {
-		// for providing ranged content
-		start, rangeLen, err = rs.GetOffsetLength(totalObjectSize)
-		if err != nil {
-			return err
-		}
+		rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
+	}
+
+	// For providing ranged content
+	start, rangeLen, err = rs.GetOffsetLength(totalObjectSize)
+	if err != nil {
+		return err
 	}
 
 	// Set content length.
@@ -197,6 +194,9 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 				fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID),
 			}
 		}
+		if objInfo.TransitionStatus == lifecycle.TransitionComplete {
+			w.Header()[xhttp.AmzStorageClass] = []string{objInfo.StorageClass}
+		}
 	}
 
 	return nil

cmd/api-response.go

@@ -35,11 +35,11 @@ import (
 
 const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
-	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
-	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
-	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
-	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
+	iso8601TimeFormat = "2006-01-02T15:04:05.000Z"                     // Reply date format with nanosecond precision.
+	maxObjectList     = metacacheBlockSize - (metacacheBlockSize / 10) // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxDeleteList     = 10000                                          // Limit number of objects deleted in a delete call.
+	maxUploadsList    = 10000                                          // Limit number of uploads in a listUploadsResponse.
+	maxPartsList      = 10000                                          // Limit number of parts in a listPartsResponse.
 )
 
 // LocationResponse - format for location response.
@@ -388,7 +388,7 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
 	}
 	proto := handlers.GetSourceScheme(r)
 	if proto == "" {
-		proto = getURLScheme(globalIsSSL)
+		proto = getURLScheme(globalIsTLS)
 	}
 	u := &url.URL{
 		Host:   r.Host,

cmd/api-router.go

@@ -27,23 +27,41 @@ import (
 )
 
 func newHTTPServerFn() *xhttp.Server {
-	globalObjLayerMutex.Lock()
-	defer globalObjLayerMutex.Unlock()
+	globalObjLayerMutex.RLock()
+	defer globalObjLayerMutex.RUnlock()
 	return globalHTTPServer
 }
 
-func newObjectLayerFn() ObjectLayer {
+func setHTTPServer(h *xhttp.Server) {
 	globalObjLayerMutex.Lock()
-	defer globalObjLayerMutex.Unlock()
+	globalHTTPServer = h
+	globalObjLayerMutex.Unlock()
+}
+
+func newObjectLayerFn() ObjectLayer {
+	globalObjLayerMutex.RLock()
+	defer globalObjLayerMutex.RUnlock()
 	return globalObjectAPI
 }
 
 func newCachedObjectLayerFn() CacheObjectLayer {
-	globalObjLayerMutex.Lock()
-	defer globalObjLayerMutex.Unlock()
+	globalObjLayerMutex.RLock()
+	defer globalObjLayerMutex.RUnlock()
 	return globalCacheObjectAPI
 }
 
+func setCacheObjectLayer(c CacheObjectLayer) {
+	globalObjLayerMutex.Lock()
+	globalCacheObjectAPI = c
+	globalObjLayerMutex.Unlock()
+}
+
+func setObjectLayer(o ObjectLayer) {
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = o
+	globalObjLayerMutex.Unlock()
+}
+
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
@@ -102,225 +120,229 @@ func registerAPIRouter(router *mux.Router) {
 		// Object operations
 		// HeadObject
 		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler))))
+			collectAPIStats("headobject", maxClients(httpTraceAll(api.HeadObjectHandler))))
 		// CopyObjectPart
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").
 			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
-			HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)))).
+			HandlerFunc(collectAPIStats("copyobjectpart", maxClients(httpTraceAll(api.CopyObjectPartHandler)))).
 			Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+			collectAPIStats("putobjectpart", maxClients(httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("listobjectparts", httpTraceAll(api.ListObjectPartsHandler)))).Queries("uploadId", "{uploadId:.*}")
+			collectAPIStats("listobjectparts", maxClients(httpTraceAll(api.ListObjectPartsHandler)))).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("completemutipartupload", httpTraceAll(api.CompleteMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
+			collectAPIStats("completemutipartupload", maxClients(httpTraceAll(api.CompleteMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("newmultipartupload", httpTraceAll(api.NewMultipartUploadHandler)))).Queries("uploads", "")
+			collectAPIStats("newmultipartupload", maxClients(httpTraceAll(api.NewMultipartUploadHandler)))).Queries("uploads", "")
 		// AbortMultipartUpload
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
+			collectAPIStats("abortmultipartupload", maxClients(httpTraceAll(api.AbortMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler)))).Queries("acl", "")
+			collectAPIStats("getobjectacl", maxClients(httpTraceHdrs(api.GetObjectACLHandler)))).Queries("acl", "")
 		// PutObjectACL - this is a dummy call.
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectacl", httpTraceHdrs(api.PutObjectACLHandler)))).Queries("acl", "")
+			collectAPIStats("putobjectacl", maxClients(httpTraceHdrs(api.PutObjectACLHandler)))).Queries("acl", "")
 		// GetObjectTagging
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("getobjecttagging", maxClients(httpTraceHdrs(api.GetObjectTaggingHandler)))).Queries("tagging", "")
 		// PutObjectTagging
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjecttagging", httpTraceHdrs(api.PutObjectTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("putobjecttagging", maxClients(httpTraceHdrs(api.PutObjectTaggingHandler)))).Queries("tagging", "")
 		// DeleteObjectTagging
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("deleteobjecttagging", httpTraceHdrs(api.DeleteObjectTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("deleteobjecttagging", maxClients(httpTraceHdrs(api.DeleteObjectTaggingHandler)))).Queries("tagging", "")
 		// SelectObjectContent
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler)))).Queries("select", "").Queries("select-type", "2")
+			collectAPIStats("selectobjectcontent", maxClients(httpTraceHdrs(api.SelectObjectContentHandler)))).Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjectretention", httpTraceAll(api.GetObjectRetentionHandler)))).Queries("retention", "")
+			collectAPIStats("getobjectretention", maxClients(httpTraceAll(api.GetObjectRetentionHandler)))).Queries("retention", "")
 		// GetObjectLegalHold
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjectlegalhold", httpTraceAll(api.GetObjectLegalHoldHandler)))).Queries("legal-hold", "")
+			collectAPIStats("getobjectlegalhold", maxClients(httpTraceAll(api.GetObjectLegalHoldHandler)))).Queries("legal-hold", "")
 		// GetObject
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler))))
+			collectAPIStats("getobject", maxClients(httpTraceHdrs(api.GetObjectHandler))))
 		// CopyObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
-			HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler))))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(
+			collectAPIStats("copyobject", maxClients(httpTraceAll(api.CopyObjectHandler))))
 		// PutObjectRetention
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")
+			collectAPIStats("putobjectretention", maxClients(httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")
 		// PutObjectLegalHold
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectlegalhold", httpTraceAll(api.PutObjectLegalHoldHandler)))).Queries("legal-hold", "")
+			collectAPIStats("putobjectlegalhold", maxClients(httpTraceAll(api.PutObjectLegalHoldHandler)))).Queries("legal-hold", "")
 
 		// PutObject
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler))))
+			collectAPIStats("putobject", maxClients(httpTraceHdrs(api.PutObjectHandler))))
 		// DeleteObject
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler))))
+			collectAPIStats("deleteobject", maxClients(httpTraceAll(api.DeleteObjectHandler))))
 
 		/// Bucket operations
 		// GetBucketLocation
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlocation", httpTraceAll(api.GetBucketLocationHandler)))).Queries("location", "")
+			collectAPIStats("getbucketlocation", maxClients(httpTraceAll(api.GetBucketLocationHandler)))).Queries("location", "")
 		// GetBucketPolicy
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler)))).Queries("policy", "")
+			collectAPIStats("getbucketpolicy", maxClients(httpTraceAll(api.GetBucketPolicyHandler)))).Queries("policy", "")
 		// GetBucketLifecycle
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("getbucketlifecycle", maxClients(httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// GetBucketEncryption
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketencryption", httpTraceAll(api.GetBucketEncryptionHandler)))).Queries("encryption", "")
+			collectAPIStats("getbucketencryption", maxClients(httpTraceAll(api.GetBucketEncryptionHandler)))).Queries("encryption", "")
 		// GetBucketObjectLockConfig
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketobjectlockconfiguration", httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "")
+			collectAPIStats("getbucketobjectlockconfiguration", maxClients(httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "")
 		// GetBucketReplicationConfig
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketreplicationconfiguration", httpTraceAll(api.GetBucketReplicationConfigHandler)))).Queries("replication", "")
+			collectAPIStats("getbucketreplicationconfiguration", maxClients(httpTraceAll(api.GetBucketReplicationConfigHandler)))).Queries("replication", "")
 
 		// GetBucketVersioning
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketversioning", httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "")
+			collectAPIStats("getbucketversioning", maxClients(httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "")
 		// GetBucketNotification
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "")
+			collectAPIStats("getbucketnotification", maxClients(httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "")
 		// ListenNotification
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listennotification", httpTraceAll(api.ListenNotificationHandler))).Queries("events", "{events:.*}")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			collectAPIStats("listennotification", maxClients(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler)))).Queries("acl", "")
+			collectAPIStats("getbucketacl", maxClients(httpTraceAll(api.GetBucketACLHandler)))).Queries("acl", "")
 		// PutBucketACL -- this is a dummy call.
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketacl", httpTraceAll(api.PutBucketACLHandler)))).Queries("acl", "")
+			collectAPIStats("putbucketacl", maxClients(httpTraceAll(api.PutBucketACLHandler)))).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler)))).Queries("cors", "")
+			collectAPIStats("getbucketcors", maxClients(httpTraceAll(api.GetBucketCorsHandler)))).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketwebsite", httpTraceAll(api.GetBucketWebsiteHandler)))).Queries("website", "")
+			collectAPIStats("getbucketwebsite", maxClients(httpTraceAll(api.GetBucketWebsiteHandler)))).Queries("website", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketaccelerate", httpTraceAll(api.GetBucketAccelerateHandler)))).Queries("accelerate", "")
+			collectAPIStats("getbucketaccelerate", maxClients(httpTraceAll(api.GetBucketAccelerateHandler)))).Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketrequestpayment", httpTraceAll(api.GetBucketRequestPaymentHandler)))).Queries("requestPayment", "")
+			collectAPIStats("getbucketrequestpayment", maxClients(httpTraceAll(api.GetBucketRequestPaymentHandler)))).Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlogging", httpTraceAll(api.GetBucketLoggingHandler)))).Queries("logging", "")
+			collectAPIStats("getbucketlogging", maxClients(httpTraceAll(api.GetBucketLoggingHandler)))).Queries("logging", "")
 		// GetBucketLifecycleHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("getbucketlifecycle", maxClients(httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// GetBucketTaggingHandler
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("getbuckettagging", maxClients(httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "")
 		//DeleteBucketWebsiteHandler
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketwebsite", httpTraceAll(api.DeleteBucketWebsiteHandler)))).Queries("website", "")
+			collectAPIStats("deletebucketwebsite", maxClients(httpTraceAll(api.DeleteBucketWebsiteHandler)))).Queries("website", "")
 		// DeleteBucketTaggingHandler
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebuckettagging", httpTraceAll(api.DeleteBucketTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("deletebuckettagging", maxClients(httpTraceAll(api.DeleteBucketTaggingHandler)))).Queries("tagging", "")
 
 		// ListMultipartUploads
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listmultipartuploads", httpTraceAll(api.ListMultipartUploadsHandler)))).Queries("uploads", "")
+			collectAPIStats("listmultipartuploads", maxClients(httpTraceAll(api.ListMultipartUploadsHandler)))).Queries("uploads", "")
 		// ListObjectsV2M
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectsv2M", httpTraceAll(api.ListObjectsV2MHandler)))).Queries("list-type", "2", "metadata", "true")
+			collectAPIStats("listobjectsv2M", maxClients(httpTraceAll(api.ListObjectsV2MHandler)))).Queries("list-type", "2", "metadata", "true")
 		// ListObjectsV2
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2")
+			collectAPIStats("listobjectsv2", maxClients(httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2")
 		// ListObjectVersions
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectversions", httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "")
+			collectAPIStats("listobjectversions", maxClients(httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "")
 		// ListObjectsV1 (Legacy)
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler))))
+			collectAPIStats("listobjectsv1", maxClients(httpTraceAll(api.ListObjectsV1Handler))))
 		// PutBucketLifecycle
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("putbucketlifecycle", maxClients(httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// PutBucketReplicationConfig
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketreplicationconfiguration", httpTraceAll(api.PutBucketReplicationConfigHandler)))).Queries("replication", "")
+			collectAPIStats("putbucketreplicationconfiguration", maxClients(httpTraceAll(api.PutBucketReplicationConfigHandler)))).Queries("replication", "")
 		// GetObjectRetention
 
 		// PutBucketEncryption
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketencryption", httpTraceAll(api.PutBucketEncryptionHandler)))).Queries("encryption", "")
+			collectAPIStats("putbucketencryption", maxClients(httpTraceAll(api.PutBucketEncryptionHandler)))).Queries("encryption", "")
 
 		// PutBucketPolicy
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler)))).Queries("policy", "")
+			collectAPIStats("putbucketpolicy", maxClients(httpTraceAll(api.PutBucketPolicyHandler)))).Queries("policy", "")
 
 		// PutBucketObjectLockConfig
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketobjectlockconfig", httpTraceAll(api.PutBucketObjectLockConfigHandler)))).Queries("object-lock", "")
+			collectAPIStats("putbucketobjectlockconfig", maxClients(httpTraceAll(api.PutBucketObjectLockConfigHandler)))).Queries("object-lock", "")
 		// PutBucketTaggingHandler
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbuckettagging", httpTraceAll(api.PutBucketTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("putbuckettagging", maxClients(httpTraceAll(api.PutBucketTaggingHandler)))).Queries("tagging", "")
 		// PutBucketVersioning
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketversioning", httpTraceAll(api.PutBucketVersioningHandler)))).Queries("versioning", "")
+			collectAPIStats("putbucketversioning", maxClients(httpTraceAll(api.PutBucketVersioningHandler)))).Queries("versioning", "")
 		// PutBucketNotification
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketnotification", httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "")
+			collectAPIStats("putbucketnotification", maxClients(httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "")
 		// PutBucket
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucket", httpTraceAll(api.PutBucketHandler))))
+			collectAPIStats("putbucket", maxClients(httpTraceAll(api.PutBucketHandler))))
 		// HeadBucket
 		bucket.Methods(http.MethodHead).HandlerFunc(
-			maxClients(collectAPIStats("headbucket", httpTraceAll(api.HeadBucketHandler))))
+			collectAPIStats("headbucket", maxClients(httpTraceAll(api.HeadBucketHandler))))
 		// PostPolicy
 		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(
-			maxClients(collectAPIStats("postpolicybucket", httpTraceHdrs(api.PostPolicyBucketHandler))))
+			collectAPIStats("postpolicybucket", maxClients(httpTraceHdrs(api.PostPolicyBucketHandler))))
 		// DeleteMultipleObjects
 		bucket.Methods(http.MethodPost).HandlerFunc(
-			maxClients(collectAPIStats("deletemultipleobjects", httpTraceAll(api.DeleteMultipleObjectsHandler)))).Queries("delete", "")
+			collectAPIStats("deletemultipleobjects", maxClients(httpTraceAll(api.DeleteMultipleObjectsHandler)))).Queries("delete", "")
 		// DeleteBucketPolicy
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler)))).Queries("policy", "")
+			collectAPIStats("deletebucketpolicy", maxClients(httpTraceAll(api.DeleteBucketPolicyHandler)))).Queries("policy", "")
 		// DeleteBucketReplication
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketreplicationconfiguration", httpTraceAll(api.DeleteBucketReplicationConfigHandler)))).Queries("replication", "")
+			collectAPIStats("deletebucketreplicationconfiguration", maxClients(httpTraceAll(api.DeleteBucketReplicationConfigHandler)))).Queries("replication", "")
 		// DeleteBucketLifecycle
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("deletebucketlifecycle", maxClients(httpTraceAll(api.DeleteBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// DeleteBucketEncryption
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketencryption", httpTraceAll(api.DeleteBucketEncryptionHandler)))).Queries("encryption", "")
+			collectAPIStats("deletebucketencryption", maxClients(httpTraceAll(api.DeleteBucketEncryptionHandler)))).Queries("encryption", "")
 		// DeleteBucket
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler))))
+			collectAPIStats("deletebucket", maxClients(httpTraceAll(api.DeleteBucketHandler))))
+		// PostRestoreObject
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
+			collectAPIStats("restoreobject", maxClients(httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "")
 	}
 
 	/// Root operation
 
 	// ListenNotification
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		collectAPIStats("listennotification", httpTraceAll(api.ListenNotificationHandler))).Queries("events", "{events:.*}")
+		collectAPIStats("listennotification", maxClients(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
 
 	// ListBuckets
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
+		collectAPIStats("listbuckets", maxClients(httpTraceAll(api.ListBucketsHandler))))
 
 	// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
 	// than failing with UnknownAPIRequest we simply handle it for now.
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
-		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
+		collectAPIStats("listbuckets", maxClients(httpTraceAll(api.ListBucketsHandler))))
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))
-	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler))
+	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3")))
 
 }
 

cmd/auth-handler.go

@@ -151,9 +151,10 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 	return cred, claims, owner, ErrNone
 }
 
-// checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
-// It does not accept presigned or JWT or anonymous requests.
-func checkAdminRequestAuthType(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
+// checkAdminRequestAuth checks for authentication and authorization for the incoming
+// request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests
+// are automatically rejected.
+func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
 	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, region)
 	if s3Err != ErrNone {
 		return cred, s3Err
@@ -333,6 +334,9 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Populate payload again to handle it in HTTP handler.
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
+	if cred.AccessKey != "" {
+		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+	}
 
 	if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
 		// Anonymous checks are not meant for ListBuckets action
@@ -464,16 +468,6 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 	return ErrNone
 }
 
-// authHandler - handles all the incoming authorization headers and validates them if possible.
-type authHandler struct {
-	handler http.Handler
-}
-
-// setAuthHandler to validate authorization header for the incoming request.
-func setAuthHandler(h http.Handler) http.Handler {
-	return authHandler{h}
-}
-
 // List of all support S3 auth types.
 var supportedS3AuthTypes = map[authType]struct{}{
 	authTypeAnonymous:       {},
@@ -491,26 +485,29 @@ func isSupportedS3AuthType(aType authType) bool {
 	return ok
 }
 
-// handler for validating incoming authorization headers.
-func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	aType := getRequestAuthType(r)
-	if isSupportedS3AuthType(aType) {
-		// Let top level caller validate for anonymous and known signed requests.
-		a.handler.ServeHTTP(w, r)
-		return
-	} else if aType == authTypeJWT {
-		// Validate Authorization header if its valid for JWT request.
-		if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
-			w.WriteHeader(http.StatusUnauthorized)
+// setAuthHandler to validate authorization header for the incoming request.
+func setAuthHandler(h http.Handler) http.Handler {
+	// handler for validating incoming authorization headers.
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		aType := getRequestAuthType(r)
+		if isSupportedS3AuthType(aType) {
+			// Let top level caller validate for anonymous and known signed requests.
+			h.ServeHTTP(w, r)
+			return
+		} else if aType == authTypeJWT {
+			// Validate Authorization header if its valid for JWT request.
+			if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			h.ServeHTTP(w, r)
+			return
+		} else if aType == authTypeSTS {
+			h.ServeHTTP(w, r)
 			return
 		}
-		a.handler.ServeHTTP(w, r)
-		return
-	} else if aType == authTypeSTS {
-		a.handler.ServeHTTP(w, r)
-		return
-	}
-	writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
+	})
 }
 
 func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool, map[string]interface{}, APIErrorCode) {
@@ -616,7 +613,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 // isPutActionAllowed - check if PUT operation is allowed on the resource, this
 // call verifies bucket policies and IAM policies, supports multi user
 // checks etc.
-func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
+func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	switch atype {
@@ -637,6 +634,10 @@ func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.R
 		return s3Err
 	}
 
+	if cred.AccessKey != "" {
+		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+	}
+
 	// Do not check for PutObjectRetentionAction permission,
 	// if mode and retain until date are not set.
 	// Can happen when bucket has default lock config set

cmd/auth-handler_test.go

@@ -421,7 +421,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 	}
 	ctx := context.Background()
 	for i, testCase := range testCases {
-		if _, s3Error := checkAdminRequestAuthType(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}

cmd/background-heal-ops.go

@@ -18,9 +18,9 @@ package cmd
 
 import (
 	"context"
-	"path"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -54,20 +54,33 @@ func (h *healRoutine) queueHealTask(task healTask) {
 	h.tasks <- task
 }
 
-func waitForLowHTTPReq(tolerance int32, maxWait time.Duration) {
-	const wait = 10 * time.Millisecond
-	waitCount := maxWait / wait
+func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
+	// No need to wait run at full speed.
+	if maxIO <= 0 {
+		return
+	}
+
+	// At max 10 attempts to wait with 100 millisecond interval before proceeding
+	waitCount := 10
+	waitTick := 100 * time.Millisecond
 
 	// Bucket notification and http trace are not costly, it is okay to ignore them
 	// while counting the number of concurrent connections
-	tolerance += int32(globalHTTPListen.NumSubscribers() + globalHTTPTrace.NumSubscribers())
+	maxIOFn := func() int {
+		return maxIO + int(globalHTTPListen.NumSubscribers()) + int(globalHTTPTrace.NumSubscribers())
+	}
 
 	if httpServer := newHTTPServerFn(); httpServer != nil {
 		// Any requests in progress, delay the heal.
-		for (httpServer.GetRequestCount() >= tolerance) &&
-			waitCount > 0 {
+		for httpServer.GetRequestCount() >= maxIOFn() {
+			time.Sleep(waitTick)
 			waitCount--
-			time.Sleep(wait)
+			if waitCount == 0 {
+				if intDataUpdateTracker.debug {
+					logger.Info("waitForLowHTTPReq: waited %d times, resuming", waitCount)
+				}
+				break
+			}
 		}
 	}
 }
@@ -81,9 +94,6 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 				break
 			}
 
-			// Wait and proceed if there are active requests
-			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()), time.Second)
-
 			var res madmin.HealResultItem
 			var err error
 			switch {
@@ -92,13 +102,10 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 			case task.bucket == SlashSeparator:
 				res, err = healDiskFormat(ctx, objAPI, task.opts)
 			case task.bucket != "" && task.object == "":
-				res, err = objAPI.HealBucket(ctx, task.bucket, task.opts.DryRun, task.opts.Remove)
+				res, err = objAPI.HealBucket(ctx, task.bucket, task.opts)
 			case task.bucket != "" && task.object != "":
 				res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
 			}
-			if task.bucket != "" && task.object != "" {
-				ObjectPathUpdated(path.Join(task.bucket, task.object))
-			}
 			task.responseCh <- healResult{result: res, err: err}
 
 		case <-h.doneCh:

cmd/background-newdisks-heal-ops.go

@@ -20,10 +20,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sort"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/console"
 )
 
 const (
@@ -39,23 +41,14 @@ type healingTracker struct {
 }
 
 func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
-	z, ok := objAPI.(*erasureServerSets)
+	z, ok := objAPI.(*erasureServerPools)
 	if !ok {
 		return
 	}
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
 
-	var bgSeq *healSequence
-	var found bool
-
-	for {
-		bgSeq, found = globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
-		if found {
-			break
-		}
-		time.Sleep(time.Second)
-	}
+	bgSeq := mustGetHealSequence(ctx)
 
 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 
@@ -64,7 +57,8 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 			drivesToHeal, defaultMonitorNewDiskInterval))
 
 		// Heal any disk format and metadata early, if possible.
-		if err := bgSeq.healDiskMeta(); err != nil {
+		// Start with format healing
+		if err := bgSeq.healDiskFormat(); err != nil {
 			if newObjectLayerFn() != nil {
 				// log only in situations, when object layer
 				// has fully initialized.
@@ -73,6 +67,14 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}
 
+	if err := bgSeq.healDiskMeta(objAPI); err != nil {
+		if newObjectLayerFn() != nil {
+			// log only in situations, when object layer
+			// has fully initialized.
+			logger.LogIf(bgSeq.ctx, err)
+		}
+	}
+
 	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
 }
 
@@ -101,20 +103,24 @@ func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	globalBackgroundHealRoutine = newHealRoutine()
 	go globalBackgroundHealRoutine.run(ctx, objAPI)
 
-	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence())
+	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence(), objAPI)
 }
 
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *healSequence) {
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
+	diskCheckTimer := time.NewTimer(defaultMonitorNewDiskInterval)
+	defer diskCheckTimer.Stop()
+wait:
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-time.After(defaultMonitorNewDiskInterval):
-			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()), time.Second)
+		case <-diskCheckTimer.C:
+			// Reset to next interval.
+			diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
 
 			var erasureSetInZoneDisksToHeal []map[int][]StorageAPI
 
@@ -129,12 +135,16 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *
 				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
 					len(healDisks)))
 
-				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.serverSets))
-				for i := range z.serverSets {
+				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
+				for i := range z.serverPools {
 					erasureSetInZoneDisksToHeal[i] = map[int][]StorageAPI{}
 				}
 			}
 
+			if serverDebugLog {
+				console.Debugf("disk check timer fired, attempting to heal %d drives\n", len(healDisks))
+			}
+
 			// heal only if new disks found.
 			for _, endpoint := range healDisks {
 				disk, format, err := connectEndpoint(endpoint)
@@ -143,40 +153,66 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *
 					continue
 				}
 
-				zoneIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
-				if zoneIdx < 0 {
+				poolIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
+				if poolIdx < 0 {
 					continue
 				}
 
 				// Calculate the set index where the current endpoint belongs
-				z.serverSets[zoneIdx].erasureDisksMu.RLock()
+				z.serverPools[poolIdx].erasureDisksMu.RLock()
 				// Protect reading reference format.
-				setIndex, _, err := findDiskIndex(z.serverSets[zoneIdx].format, format)
-				z.serverSets[zoneIdx].erasureDisksMu.RUnlock()
+				setIndex, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
+				z.serverPools[poolIdx].erasureDisksMu.RUnlock()
 				if err != nil {
 					printEndpointError(endpoint, err, false)
 					continue
 				}
 
-				erasureSetInZoneDisksToHeal[zoneIdx][setIndex] = append(erasureSetInZoneDisksToHeal[zoneIdx][setIndex], disk)
+				erasureSetInZoneDisksToHeal[poolIdx][setIndex] = append(erasureSetInZoneDisksToHeal[poolIdx][setIndex], disk)
 			}
 
-			buckets, _ := z.ListBucketsHeal(ctx)
+			buckets, _ := z.ListBuckets(ctx)
+
+			// Heal latest buckets first.
+			sort.Slice(buckets, func(i, j int) bool {
+				return buckets[i].Created.After(buckets[j].Created)
+			})
+
 			for i, setMap := range erasureSetInZoneDisksToHeal {
 				for setIndex, disks := range setMap {
 					for _, disk := range disks {
-						logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
+						logger.Info("Healing disk '%s' on %s pool", disk, humanize.Ordinal(i+1))
 
-						lbDisks := z.serverSets[i].sets[setIndex].getOnlineDisks()
+						// So someone changed the drives underneath, healing tracker missing.
+						if !disk.Healing() {
+							logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool", disk, humanize.Ordinal(i+1))
+							diskID, err := disk.GetDiskID()
+							if err != nil {
+								logger.LogIf(ctx, err)
+								// reading format.json failed or not found, proceed to look
+								// for new disks to be healed again, we cannot proceed further.
+								goto wait
+							}
+
+							if err := saveHealingTracker(disk, diskID); err != nil {
+								logger.LogIf(ctx, err)
+								// Unable to write healing tracker, permission denied or some
+								// other unexpected error occurred. Proceed to look for new
+								// disks to be healed again, we cannot proceed further.
+								goto wait
+							}
+						}
+
+						lbDisks := z.serverPools[i].sets[setIndex].getOnlineDisks()
 						if err := healErasureSet(ctx, setIndex, buckets, lbDisks); err != nil {
 							logger.LogIf(ctx, err)
 							continue
 						}
 
-						logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1))
+						logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))
 
-						if err := disk.DeleteFile(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
-							healingTrackerFilename); err != nil && !errors.Is(err, errFileNotFound) {
+						if err := disk.Delete(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
+							healingTrackerFilename, false); err != nil && !errors.Is(err, errFileNotFound) {
 							logger.LogIf(ctx, err)
 							continue
 						}

cmd/benchmark-utils_test.go

@@ -55,7 +55,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < b.N; i++ {
 		// insert the object.
 		objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -114,7 +114,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			md5hex := getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
 			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
-				mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
+				mustGetPutObjReader(b, bytes.NewReader(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -200,7 +200,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -301,7 +301,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		for pb.Next() {
 			// insert the object.
 			objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-				mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
+				mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -340,7 +340,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}

cmd/bitrot-streaming.go

@@ -91,7 +91,8 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 // ReadAt() implementation which verifies the bitrot hash available as part of the stream.
 type streamingBitrotReader struct {
 	disk       StorageAPI
-	rc         io.ReadCloser
+	data       []byte
+	rc         io.Reader
 	volume     string
 	filePath   string
 	tillOffset int64
@@ -105,7 +106,10 @@ func (b *streamingBitrotReader) Close() error {
 	if b.rc == nil {
 		return nil
 	}
-	return b.rc.Close()
+	if closer, ok := b.rc.(io.Closer); ok {
+		return closer.Close()
+	}
+	return nil
 }
 
 func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
@@ -119,11 +123,16 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		// For the first ReadAt() call we need to open the stream for reading.
 		b.currOffset = offset
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
-		b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
+		if len(b.data) == 0 {
+			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
+		} else {
+			b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
+		}
 		if err != nil {
 			return 0, err
 		}
 	}
+
 	if offset != b.currOffset {
 		// Can never happen unless there are programmer bugs
 		return 0, errUnexpected
@@ -150,10 +159,11 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 }
 
 // Returns streaming bitrot reader implementation.
-func newStreamingBitrotReader(disk StorageAPI, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
+func newStreamingBitrotReader(disk StorageAPI, data []byte, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
 	h := algo.New()
 	return &streamingBitrotReader{
 		disk,
+		data,
 		nil,
 		volume,
 		filePath,

cmd/bitrot.go

@@ -103,9 +103,9 @@ func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, alg
 	return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
 }
 
-func newBitrotReader(disk StorageAPI, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
+func newBitrotReader(disk StorageAPI, data []byte, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
 	if algo == HighwayHash256S {
-		return newStreamingBitrotReader(disk, bucket, filePath, tillOffset, algo, shardSize)
+		return newStreamingBitrotReader(disk, data, bucket, filePath, tillOffset, algo, shardSize)
 	}
 	return newWholeBitrotReader(disk, bucket, filePath, algo, tillOffset, sum)
 }

cmd/bitrot_test.go

@@ -62,7 +62,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	}
 	writer.(io.Closer).Close()
 
-	reader := newBitrotReader(disk, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
+	reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
 	b := make([]byte, 10)
 	if _, err = reader.ReadAt(b, 0); err != nil {
 		log.Fatal(err)

cmd/bootstrap-peer-server.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -53,7 +52,7 @@ type bootstrapRESTServer struct{}
 type ServerSystemConfig struct {
 	MinioPlatform  string
 	MinioRuntime   string
-	MinioEndpoints EndpointServerSets
+	MinioEndpoints EndpointServerPools
 }
 
 // Diff - returns error on first difference found in two configs.
@@ -160,9 +159,9 @@ func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg ServerSyst
 	return srcCfg.Diff(recvCfg)
 }
 
-func verifyServerSystemConfig(ctx context.Context, endpointServerSets EndpointServerSets) error {
+func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointServerPools) error {
 	srcCfg := getServerSystemCfg()
-	clnts := newBootstrapRESTClients(endpointServerSets)
+	clnts := newBootstrapRESTClients(endpointServerPools)
 	var onlineServers int
 	var offlineEndpoints []string
 	var retries int
@@ -197,10 +196,10 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerSets EndpointSe
 	return nil
 }
 
-func newBootstrapRESTClients(endpointServerSets EndpointServerSets) []*bootstrapRESTClient {
+func newBootstrapRESTClients(endpointServerPools EndpointServerPools) []*bootstrapRESTClient {
 	seenHosts := set.NewStringSet()
 	var clnts []*bootstrapRESTClient
-	for _, ep := range endpointServerSets {
+	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
 			if seenHosts.Contains(endpoint.Host) {
 				continue
@@ -224,16 +223,7 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
 		Path:   bootstrapRESTPath,
 	}
 
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: endpoint.Hostname(),
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
-	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
+	restClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken)
 	restClient.HealthCheckFn = nil
 
 	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient}

cmd/bucket-handlers.go

@@ -28,19 +28,19 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/bucket/replication"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
@@ -383,6 +383,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
+	// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
+	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
+
 	// Before proceeding validate if bucket exists.
 	_, err := objectAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
@@ -400,7 +404,18 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	if api.CacheAPI() != nil {
 		getObjectInfoFn = api.CacheAPI().GetObjectInfo
 	}
-
+	var (
+		hasLockEnabled, hasLifecycleConfig, replicateSync bool
+		goi                                               ObjectInfo
+		gerr                                              error
+	)
+	replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjects.Objects)
+	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
+		hasLockEnabled = true
+	}
+	if _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket); err == nil {
+		hasLifecycleConfig = true
+	}
 	dErrs := make([]DeleteError, len(deleteObjects.Objects))
 	for index, object := range deleteObjects.Objects {
 		if apiErrCode := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); apiErrCode != ErrNone {
@@ -417,10 +432,55 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			}
 			continue
 		}
+		if object.VersionID != "" && object.VersionID != nullVersionID {
+			if _, err := uuid.Parse(object.VersionID); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("invalid version-id specified %w", err))
+				apiErr := errorCodes.ToAPIErr(ErrNoSuchVersion)
+				dErrs[index] = DeleteError{
+					Code:      apiErr.Code,
+					Message:   apiErr.Description,
+					Key:       object.ObjectName,
+					VersionID: object.VersionID,
+				}
+				continue
+			}
+		}
 
+		if replicateDeletes || hasLockEnabled || hasLifecycleConfig {
+			goi, gerr = getObjectInfoFn(ctx, bucket, object.ObjectName, ObjectOptions{
+				VersionID: object.VersionID,
+			})
+		}
+		if hasLifecycleConfig && gerr == nil {
+			object.PurgeTransitioned = goi.TransitionStatus
+		}
+		if replicateDeletes {
+			delMarker, replicate, repsync := checkReplicateDelete(ctx, bucket, ObjectToDelete{
+				ObjectName: object.ObjectName,
+				VersionID:  object.VersionID,
+			}, goi, gerr)
+			replicateSync = repsync
+			if replicate {
+				if apiErrCode := checkRequestAuthType(ctx, r, policy.ReplicateDeleteAction, bucket, object.ObjectName); apiErrCode != ErrNone {
+					if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
+						writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL, guessIsBrowserReq(r))
+						return
+					}
+					continue
+				}
+				if object.VersionID != "" {
+					object.VersionPurgeStatus = Pending
+					if delMarker {
+						object.DeleteMarkerVersionID = object.VersionID
+					}
+				} else {
+					object.DeleteMarkerReplicationStatus = string(replication.Pending)
+				}
+			}
+		}
 		if object.VersionID != "" {
-			if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
-				if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, getObjectInfoFn); apiErrCode != ErrNone {
+			if hasLockEnabled {
+				if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); apiErrCode != ErrNone {
 					apiErr := errorCodes.ToAPIErr(apiErrCode)
 					dErrs[index] = DeleteError{
 						Code:      apiErr.Code,
@@ -454,15 +514,25 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		Versioned:        globalBucketVersioningSys.Enabled(bucket),
 		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
-
 	deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
 	for i := range errs {
-		dindex := objectsToDelete[deleteList[i]]
-		apiErr := toAPIError(ctx, errs[i])
-		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" || apiErr.Code == "InvalidArgument" {
+		dindex := objectsToDelete[ObjectToDelete{
+			ObjectName:                    dObjects[i].ObjectName,
+			VersionID:                     dObjects[i].VersionID,
+			DeleteMarkerVersionID:         dObjects[i].DeleteMarkerVersionID,
+			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus,
+			DeleteMarkerReplicationStatus: dObjects[i].DeleteMarkerReplicationStatus,
+			PurgeTransitioned:             dObjects[i].PurgeTransitioned,
+		}]
+		if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) {
+			if replicateDeletes {
+				dObjects[i].DeleteMarkerReplicationStatus = deleteList[i].DeleteMarkerReplicationStatus
+				dObjects[i].VersionPurgeStatus = deleteList[i].VersionPurgeStatus
+			}
 			deletedObjects[dindex] = dObjects[i]
 			continue
 		}
+		apiErr := toAPIError(ctx, errs[i])
 		dErrs[dindex] = DeleteError{
 			Code:      apiErr.Code,
 			Message:   apiErr.Description,
@@ -484,7 +554,30 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)
+	for _, dobj := range deletedObjects {
+		if replicateDeletes {
+			if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
+				dv := DeletedObjectVersionInfo{
+					DeletedObject: dobj,
+					Bucket:        bucket,
+				}
+				scheduleReplicationDelete(ctx, dv, objectAPI, replicateSync)
+			}
+		}
 
+		if hasLifecycleConfig && dobj.PurgeTransitioned == lifecycle.TransitionComplete { // clean up transitioned tier
+			action := lifecycle.DeleteAction
+			if dobj.VersionID != "" {
+				action = lifecycle.DeleteVersionAction
+			}
+			deleteTransitionedObject(ctx, newObjectLayerFn(), bucket, dobj.ObjectName, lifecycle.ObjectOpts{
+				Name:         dobj.ObjectName,
+				VersionID:    dobj.VersionID,
+				DeleteMarker: dobj.DeleteMarker,
+			}, action, true)
+		}
+
+	}
 	// Notify deleted event for objects.
 	for _, dobj := range deletedObjects {
 		eventName := event.ObjectRemovedDelete
@@ -662,7 +755,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
+	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -809,7 +902,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if _, err = globalBucketSSEConfigSys.Get(bucket); err == nil || globalAutoEncryption {
 		// This request header needs to be set prior to setting ObjectOptions
 		if !crypto.SSEC.IsRequested(r.Header) {
-			r.Header.Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+			r.Header.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 		}
 	}
 
@@ -821,7 +914,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if crypto.IsRequested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
+		if _, ok := crypto.IsRequested(formValues); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
 			if crypto.SSECopy.IsRequested(r.Header) {
 				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
 				return
@@ -1250,11 +1343,6 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
-	// Turn off replication if disk crawl is unavailable.
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketReplicationDisabledError), r.URL)
-		return
-	}
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
@@ -1291,7 +1379,6 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
 	if err = globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/bucket-handlers_test.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"encoding/xml"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -36,7 +35,7 @@ func TestRemoveBucketHandler(t *testing.T) {
 
 func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
-	_, err := obj.PutObject(GlobalContext, bucketName, "test-object", mustGetPutObjReader(t, bytes.NewBuffer([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{})
+	_, err := obj.PutObject(GlobalContext, bucketName, "test-object", mustGetPutObjReader(t, bytes.NewReader([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{})
 	// if object upload fails stop the test.
 	if err != nil {
 		t.Fatalf("Error uploading object: <ERROR> %v", err)
@@ -670,7 +669,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
+		_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)
@@ -837,7 +836,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		// Verify whether the bucket obtained object is same as the one created.
 		if testCase.expectedContent != nil && !bytes.Equal(testCase.expectedContent, actualContent) {
-			fmt.Println(string(testCase.expectedContent), string(actualContent))
+			t.Log(string(testCase.expectedContent), string(actualContent))
 			t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i+1, instanceType)
 		}
 	}

cmd/bucket-lifecycle-handlers.go

@@ -78,6 +78,12 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Validate the transition storage ARNs
+	if err = validateLifecycleTransition(ctx, bucket, bucketLifecycle); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	configData, err := xml.Marshal(bucketLifecycle)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/bucket-lifecycle.go

@@ -17,7 +17,25 @@
 package cmd
 
 import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"runtime"
+	"strings"
+	"time"
+
+	miniogo "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	sse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/s3select"
 )
 
 const (
@@ -46,3 +64,592 @@ func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err er
 func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
+
+type transitionState struct {
+	// add future metrics here
+	transitionCh chan ObjectInfo
+}
+
+func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
+	select {
+	case t.transitionCh <- oi:
+	default:
+	}
+}
+
+var (
+	globalTransitionState      *transitionState
+	globalTransitionConcurrent = runtime.GOMAXPROCS(0) / 2
+)
+
+func newTransitionState() *transitionState {
+
+	// fix minimum concurrent transition to 1 for single CPU setup
+	if globalTransitionConcurrent == 0 {
+		globalTransitionConcurrent = 1
+	}
+	ts := &transitionState{
+		transitionCh: make(chan ObjectInfo, 10000),
+	}
+	go func() {
+		<-GlobalContext.Done()
+		close(ts.transitionCh)
+	}()
+	return ts
+}
+
+// addWorker creates a new worker to process tasks
+func (t *transitionState) addWorker(ctx context.Context, objectAPI ObjectLayer) {
+	// Add a new worker.
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case oi, ok := <-t.transitionCh:
+				if !ok {
+					return
+				}
+				if err := transitionObject(ctx, objectAPI, oi); err != nil {
+					logger.LogIf(ctx, err)
+				}
+			}
+		}
+	}()
+}
+
+func initBackgroundTransition(ctx context.Context, objectAPI ObjectLayer) {
+	if globalTransitionState == nil {
+		return
+	}
+
+	// Start with globalTransitionConcurrent.
+	for i := 0; i < globalTransitionConcurrent; i++ {
+		globalTransitionState.addWorker(ctx, objectAPI)
+	}
+}
+
+func validateLifecycleTransition(ctx context.Context, bucket string, lfc *lifecycle.Lifecycle) error {
+	for _, rule := range lfc.Rules {
+		if rule.Transition.StorageClass != "" {
+			sameTarget, destbucket, err := validateTransitionDestination(ctx, bucket, rule.Transition.StorageClass)
+			if err != nil {
+				return err
+			}
+			if sameTarget && destbucket == bucket {
+				return fmt.Errorf("Transition destination cannot be the same as the source bucket")
+			}
+		}
+	}
+	return nil
+}
+
+// validateTransitionDestination returns error if transition destination bucket missing or not configured
+// It also returns true if transition destination is same as this server.
+func validateTransitionDestination(ctx context.Context, bucket string, targetLabel string) (bool, string, error) {
+	tgt := globalBucketTargetSys.GetRemoteTargetWithLabel(ctx, bucket, targetLabel)
+	if tgt == nil {
+		return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	arn, err := madmin.ParseARN(tgt.Arn)
+	if err != nil {
+		return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	if arn.Type != madmin.ILMService {
+		return false, "", BucketRemoteArnTypeInvalid{}
+	}
+	clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, tgt.Arn)
+	if clnt == nil {
+		return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	if found, _ := clnt.BucketExists(ctx, arn.Bucket); !found {
+		return false, "", BucketRemoteDestinationNotFound{Bucket: arn.Bucket}
+	}
+	sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
+	return sameTarget, arn.Bucket, nil
+}
+
+// transitionSC returns storage class label for this bucket
+func transitionSC(ctx context.Context, bucket string) string {
+	cfg, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+	if err != nil {
+		return ""
+	}
+	for _, rule := range cfg.Rules {
+		if rule.Status == Disabled {
+			continue
+		}
+		if rule.Transition.StorageClass != "" {
+			return rule.Transition.StorageClass
+		}
+	}
+	return ""
+}
+
+// return true if ARN representing transition storage class is present in a active rule
+// for the lifecycle configured on this bucket
+func transitionSCInUse(ctx context.Context, lfc *lifecycle.Lifecycle, bucket, arnStr string) bool {
+	tgtLabel := globalBucketTargetSys.GetRemoteLabelWithArn(ctx, bucket, arnStr)
+	if tgtLabel == "" {
+		return false
+	}
+	for _, rule := range lfc.Rules {
+		if rule.Status == Disabled {
+			continue
+		}
+		if rule.Transition.StorageClass != "" && rule.Transition.StorageClass == tgtLabel {
+			return true
+		}
+	}
+	return false
+}
+
+// set PutObjectOptions for PUT operation to transition data to target cluster
+func putTransitionOpts(objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
+	meta := make(map[string]string)
+
+	tag, err := tags.ParseObjectTags(objInfo.UserTags)
+	if err != nil {
+		return
+	}
+	putOpts = miniogo.PutObjectOptions{
+		UserMetadata:    meta,
+		UserTags:        tag.ToMap(),
+		ContentType:     objInfo.ContentType,
+		ContentEncoding: objInfo.ContentEncoding,
+		StorageClass:    objInfo.StorageClass,
+		Internal: miniogo.AdvancedPutOptions{
+			SourceVersionID: objInfo.VersionID,
+			SourceMTime:     objInfo.ModTime,
+			SourceETag:      objInfo.ETag,
+		},
+	}
+	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
+		rmode := miniogo.RetentionMode(mode)
+		putOpts.Mode = rmode
+	}
+	if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
+		rdate, err := time.Parse(time.RFC3339, retainDateStr)
+		if err != nil {
+			return
+		}
+		putOpts.RetainUntilDate = rdate
+	}
+	if lhold, ok := objInfo.UserDefined[xhttp.AmzObjectLockLegalHold]; ok {
+		putOpts.LegalHold = miniogo.LegalHoldStatus(lhold)
+	}
+
+	return
+}
+
+// handle deletes of transitioned objects or object versions when one of the following is true:
+// 1. temporarily restored copies of objects (restored with the PostRestoreObject API) expired.
+// 2. life cycle expiry date is met on the object.
+// 3. Object is removed through DELETE api call
+func deleteTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket, object string, lcOpts lifecycle.ObjectOpts, action lifecycle.Action, isDeleteTierOnly bool) error {
+	if lcOpts.TransitionStatus == "" && !isDeleteTierOnly {
+		return nil
+	}
+	lc, err := globalLifecycleSys.Get(bucket)
+	if err != nil {
+		return err
+	}
+	arn := getLifecycleTransitionTargetArn(ctx, lc, bucket, lcOpts)
+	if arn == nil {
+		return fmt.Errorf("remote target not configured")
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
+	if tgt == nil {
+		return fmt.Errorf("remote target not configured")
+	}
+
+	var opts ObjectOptions
+	opts.Versioned = globalBucketVersioningSys.Enabled(bucket)
+	opts.VersionID = lcOpts.VersionID
+	switch action {
+	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		// delete locally restored copy of object or object version
+		// from the source, while leaving metadata behind. The data on
+		// transitioned tier lies untouched and still accessible
+		opts.TransitionStatus = lcOpts.TransitionStatus
+		_, err = objectAPI.DeleteObject(ctx, bucket, object, opts)
+		return err
+	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
+		// When an object is past expiry, delete the data from transitioned tier and
+		// metadata from source
+		if err := tgt.RemoveObject(context.Background(), arn.Bucket, object, miniogo.RemoveObjectOptions{VersionID: lcOpts.VersionID}); err != nil {
+			logger.LogIf(ctx, err)
+		}
+
+		if isDeleteTierOnly {
+			return nil
+		}
+		_, err = objectAPI.DeleteObject(ctx, bucket, object, opts)
+		if err != nil {
+			return err
+		}
+		eventName := event.ObjectRemovedDelete
+		if lcOpts.DeleteMarker {
+			eventName = event.ObjectRemovedDeleteMarkerCreated
+		}
+		objInfo := ObjectInfo{
+			Name:         object,
+			VersionID:    lcOpts.VersionID,
+			DeleteMarker: lcOpts.DeleteMarker,
+		}
+		// Notify object deleted event.
+		sendEvent(eventArgs{
+			EventName:  eventName,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [ILM-EXPIRY]",
+		})
+	}
+
+	// should never reach here
+	return nil
+}
+
+// transition object to target specified by the transition ARN. When an object is transitioned to another
+// storage specified by the transition ARN, the metadata is left behind on source cluster and original content
+// is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
+// to the transition tier without decrypting or re-encrypting.
+func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo ObjectInfo) error {
+	lc, err := globalLifecycleSys.Get(objInfo.Bucket)
+	if err != nil {
+		return err
+	}
+	lcOpts := lifecycle.ObjectOpts{
+		Name:     objInfo.Name,
+		UserTags: objInfo.UserTags,
+	}
+	arn := getLifecycleTransitionTargetArn(ctx, lc, objInfo.Bucket, lcOpts)
+	if arn == nil {
+		return fmt.Errorf("remote target not configured")
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
+	if tgt == nil {
+		return fmt.Errorf("remote target not configured")
+	}
+
+	gr, err := objectAPI.GetObjectNInfo(ctx, objInfo.Bucket, objInfo.Name, nil, http.Header{}, readLock, ObjectOptions{
+		VersionID:        objInfo.VersionID,
+		TransitionStatus: lifecycle.TransitionPending,
+	})
+	if err != nil {
+		return err
+	}
+	oi := gr.ObjInfo
+	if oi.TransitionStatus == lifecycle.TransitionComplete {
+		gr.Close()
+		return nil
+	}
+
+	putOpts := putTransitionOpts(oi)
+	if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, putOpts); err != nil {
+		gr.Close()
+		return err
+	}
+	gr.Close()
+
+	var opts ObjectOptions
+	opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
+	opts.VersionID = oi.VersionID
+	opts.TransitionStatus = lifecycle.TransitionComplete
+	eventName := event.ObjectTransitionComplete
+
+	_, err = objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
+	if err != nil {
+		eventName = event.ObjectTransitionFailed
+	}
+	// Notify object deleted event.
+	sendEvent(eventArgs{
+		EventName:  eventName,
+		BucketName: oi.Bucket,
+		Object: ObjectInfo{
+			Name:      oi.Name,
+			VersionID: opts.VersionID,
+		},
+		Host: "Internal: [ILM-Transition]",
+	})
+	return err
+}
+
+// getLifecycleTransitionTargetArn returns transition ARN for storage class specified in the config.
+func getLifecycleTransitionTargetArn(ctx context.Context, lc *lifecycle.Lifecycle, bucket string, obj lifecycle.ObjectOpts) *madmin.ARN {
+	for _, rule := range lc.FilterActionableRules(obj) {
+		if rule.Transition.StorageClass != "" {
+			return globalBucketTargetSys.GetRemoteArnWithLabel(ctx, bucket, rule.Transition.StorageClass)
+		}
+	}
+	return nil
+}
+
+// getTransitionedObjectReader returns a reader from the transitioned tier.
+func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, oi ObjectInfo, opts ObjectOptions) (gr *GetObjectReader, err error) {
+	var lc *lifecycle.Lifecycle
+	lc, err = globalLifecycleSys.Get(bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	arn := getLifecycleTransitionTargetArn(ctx, lc, bucket, lifecycle.ObjectOpts{
+		Name:         object,
+		UserTags:     oi.UserTags,
+		ModTime:      oi.ModTime,
+		VersionID:    oi.VersionID,
+		DeleteMarker: oi.DeleteMarker,
+		IsLatest:     oi.IsLatest,
+	})
+	if arn == nil {
+		return nil, fmt.Errorf("remote target not configured")
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
+	if tgt == nil {
+		return nil, fmt.Errorf("remote target not configured")
+	}
+	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
+	if err != nil {
+		return nil, ErrorRespToObjectError(err, bucket, object)
+	}
+	gopts := miniogo.GetObjectOptions{VersionID: opts.VersionID}
+
+	// get correct offsets for encrypted object
+	if off >= 0 && length >= 0 {
+		if err := gopts.SetRange(off, off+length-1); err != nil {
+			return nil, ErrorRespToObjectError(err, bucket, object)
+		}
+	}
+
+	reader, err := tgt.GetObject(ctx, arn.Bucket, object, gopts)
+	if err != nil {
+		return nil, err
+	}
+	closeReader := func() { reader.Close() }
+
+	return fn(reader, h, opts.CheckPrecondFn, closeReader)
+}
+
+// RestoreRequestType represents type of restore.
+type RestoreRequestType string
+
+const (
+	// SelectRestoreRequest specifies select request. This is the only valid value
+	SelectRestoreRequest RestoreRequestType = "SELECT"
+)
+
+// Encryption specifies encryption setting on restored bucket
+type Encryption struct {
+	EncryptionType sse.SSEAlgorithm `xml:"EncryptionType"`
+	KMSContext     string           `xml:"KMSContext,omitempty"`
+	KMSKeyID       string           `xml:"KMSKeyId,omitempty"`
+}
+
+// MetadataEntry denotes name and value.
+type MetadataEntry struct {
+	Name  string `xml:"Name"`
+	Value string `xml:"Value"`
+}
+
+// S3Location specifies s3 location that receives result of a restore object request
+type S3Location struct {
+	BucketName   string          `xml:"BucketName,omitempty"`
+	Encryption   Encryption      `xml:"Encryption,omitempty"`
+	Prefix       string          `xml:"Prefix,omitempty"`
+	StorageClass string          `xml:"StorageClass,omitempty"`
+	Tagging      *tags.Tags      `xml:"Tagging,omitempty"`
+	UserMetadata []MetadataEntry `xml:"UserMetadata"`
+}
+
+// OutputLocation specifies bucket where object needs to be restored
+type OutputLocation struct {
+	S3 S3Location `xml:"S3,omitempty"`
+}
+
+// IsEmpty returns true if output location not specified.
+func (o *OutputLocation) IsEmpty() bool {
+	return o.S3.BucketName == ""
+}
+
+// SelectParameters specifies sql select parameters
+type SelectParameters struct {
+	s3select.S3Select
+}
+
+// IsEmpty returns true if no select parameters set
+func (sp *SelectParameters) IsEmpty() bool {
+	return sp == nil || sp.S3Select == s3select.S3Select{}
+}
+
+var (
+	selectParamsXMLName = "SelectParameters"
+)
+
+// UnmarshalXML - decodes XML data.
+func (sp *SelectParameters) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	// Essentially the same as S3Select barring the xml name.
+	if start.Name.Local == selectParamsXMLName {
+		start.Name = xml.Name{Space: "", Local: "SelectRequest"}
+	}
+	return sp.S3Select.UnmarshalXML(d, start)
+}
+
+// RestoreObjectRequest - xml to restore a transitioned object
+type RestoreObjectRequest struct {
+	XMLName          xml.Name           `xml:"http://s3.amazonaws.com/doc/2006-03-01/ RestoreRequest" json:"-"`
+	Days             int                `xml:"Days,omitempty"`
+	Type             RestoreRequestType `xml:"Type,omitempty"`
+	Tier             string             `xml:"Tier,-"`
+	Description      string             `xml:"Description,omitempty"`
+	SelectParameters *SelectParameters  `xml:"SelectParameters,omitempty"`
+	OutputLocation   OutputLocation     `xml:"OutputLocation,omitempty"`
+}
+
+// Maximum 2MiB size per restore object request.
+const maxRestoreObjectRequestSize = 2 << 20
+
+// parseRestoreRequest parses RestoreObjectRequest from xml
+func parseRestoreRequest(reader io.Reader) (*RestoreObjectRequest, error) {
+	req := RestoreObjectRequest{}
+	if err := xml.NewDecoder(io.LimitReader(reader, maxRestoreObjectRequestSize)).Decode(&req); err != nil {
+		return nil, err
+	}
+	return &req, nil
+}
+
+// validate a RestoreObjectRequest as per AWS S3 spec https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html
+func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer) error {
+	if r.Type != SelectRestoreRequest && !r.SelectParameters.IsEmpty() {
+		return fmt.Errorf("Select parameters can only be specified with SELECT request type")
+	}
+	if r.Type == SelectRestoreRequest && r.SelectParameters.IsEmpty() {
+		return fmt.Errorf("SELECT restore request requires select parameters to be specified")
+	}
+
+	if r.Type != SelectRestoreRequest && !r.OutputLocation.IsEmpty() {
+		return fmt.Errorf("OutputLocation required only for SELECT request type")
+	}
+	if r.Type == SelectRestoreRequest && r.OutputLocation.IsEmpty() {
+		return fmt.Errorf("OutputLocation required for SELECT requests")
+	}
+
+	if r.Days != 0 && r.Type == SelectRestoreRequest {
+		return fmt.Errorf("Days cannot be specified with SELECT restore request")
+	}
+	if r.Days == 0 && r.Type != SelectRestoreRequest {
+		return fmt.Errorf("restoration days should be at least 1")
+	}
+	// Check if bucket exists.
+	if !r.OutputLocation.IsEmpty() {
+		if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName); err != nil {
+			return err
+		}
+		if r.OutputLocation.S3.Prefix == "" {
+			return fmt.Errorf("Prefix is a required parameter in OutputLocation")
+		}
+		if r.OutputLocation.S3.Encryption.EncryptionType != xhttp.AmzEncryptionAES {
+			return NotImplemented{}
+		}
+	}
+	return nil
+}
+
+// set ObjectOptions for PUT call to restore temporary copy of transitioned data
+func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) {
+	meta := make(map[string]string)
+	sc := rreq.OutputLocation.S3.StorageClass
+	if sc == "" {
+		sc = objInfo.StorageClass
+	}
+	meta[strings.ToLower(xhttp.AmzStorageClass)] = sc
+
+	if rreq.Type == SelectRestoreRequest {
+		for _, v := range rreq.OutputLocation.S3.UserMetadata {
+			if !strings.HasPrefix("x-amz-meta", strings.ToLower(v.Name)) {
+				meta["x-amz-meta-"+v.Name] = v.Value
+				continue
+			}
+			meta[v.Name] = v.Value
+		}
+		meta[xhttp.AmzObjectTagging] = rreq.OutputLocation.S3.Tagging.String()
+		if rreq.OutputLocation.S3.Encryption.EncryptionType != "" {
+			meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+		}
+		return ObjectOptions{
+			Versioned:        globalBucketVersioningSys.Enabled(bucket),
+			VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
+			UserDefined:      meta,
+		}
+	}
+	for k, v := range objInfo.UserDefined {
+		meta[k] = v
+	}
+	meta[xhttp.AmzObjectTagging] = objInfo.UserTags
+
+	return ObjectOptions{
+		Versioned:        globalBucketVersioningSys.Enabled(bucket),
+		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
+		UserDefined:      meta,
+		VersionID:        objInfo.VersionID,
+		MTime:            objInfo.ModTime,
+		Expires:          objInfo.Expires,
+	}
+}
+
+var (
+	errRestoreHDRMissing   = fmt.Errorf("x-amz-restore header not found")
+	errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
+)
+
+// parse x-amz-restore header from user metadata to get the status of ongoing request and expiry of restoration
+// if any. This header value is of format: ongoing-request=true|false, expires=time
+func parseRestoreHeaderFromMeta(meta map[string]string) (ongoing bool, expiry time.Time, err error) {
+	restoreHdr, ok := meta[xhttp.AmzRestore]
+	if !ok {
+		return ongoing, expiry, errRestoreHDRMissing
+	}
+	rslc := strings.SplitN(restoreHdr, ",", 2)
+	if len(rslc) != 2 {
+		return ongoing, expiry, errRestoreHDRMalformed
+	}
+	rstatusSlc := strings.SplitN(rslc[0], "=", 2)
+	if len(rstatusSlc) != 2 {
+		return ongoing, expiry, errRestoreHDRMalformed
+	}
+	rExpSlc := strings.SplitN(rslc[1], "=", 2)
+	if len(rExpSlc) != 2 {
+		return ongoing, expiry, errRestoreHDRMalformed
+	}
+
+	expiry, err = time.Parse(http.TimeFormat, rExpSlc[1])
+	if err != nil {
+		return
+	}
+	return rstatusSlc[1] == "true", expiry, nil
+}
+
+// restoreTransitionedObject is similar to PostObjectRestore from AWS GLACIER
+// storage class. When PostObjectRestore API is called, a temporary copy of the object
+// is restored locally to the bucket on source cluster until the restore expiry date.
+// The copy that was transitioned continues to reside in the transitioned tier.
+func restoreTransitionedObject(ctx context.Context, bucket, object string, objAPI ObjectLayer, objInfo ObjectInfo, rreq *RestoreObjectRequest, restoreExpiry time.Time) error {
+	var rs *HTTPRangeSpec
+	gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, objInfo, ObjectOptions{
+		VersionID: objInfo.VersionID})
+	if err != nil {
+		return err
+	}
+	defer gr.Close()
+	hashReader, err := hash.NewReader(gr, objInfo.Size, "", "", objInfo.Size, globalCLIContext.StrictS3Compat)
+	if err != nil {
+		return err
+	}
+	pReader := NewPutObjReader(hashReader, nil, nil)
+	opts := putRestoreOpts(bucket, object, rreq, objInfo)
+	opts.UserDefined[xhttp.AmzRestore] = fmt.Sprintf("ongoing-request=%t, expiry-date=%s", false, restoreExpiry.Format(http.TimeFormat))
+	if _, err := objAPI.PutObject(ctx, bucket, object, pReader, opts); err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/bucket-listobjects-handlers.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -114,15 +113,6 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Forward the request using Source IP or bucket
-	forwardStr := handlers.GetSourceIPFromHeaders(r)
-	if forwardStr == "" {
-		forwardStr = bucket
-	}
-	if proxyRequestByStringHash(ctx, w, r, forwardStr) {
-		return
-	}
-
 	listObjectVersions := objectAPI.ListObjectVersions
 
 	// Inititate a list object versions operation based on the input params.
@@ -145,7 +135,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 // ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 10000)
-// of the objects in a bucket. You can use the request parame<ters as selection
+// of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
 // NOTE: It is recommended that this API to be used for application development.
@@ -185,13 +175,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Analyze continuation token and route the request accordingly
-	var success bool
-	token, success = proxyRequestByToken(ctx, w, r, token)
-	if success {
-		return
-	}
-
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -207,9 +190,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
-		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
-	}
 
 	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
@@ -262,13 +242,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Analyze continuation token and route the request accordingly
-	var success bool
-	token, success = proxyRequestByToken(ctx, w, r, token)
-	if success {
-		return
-	}
-
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -282,13 +255,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
 
-	// The next continuation token has id@node_index format to optimize paginated listing
-	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
-		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
-	}
-
-	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
+	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
 		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
 
@@ -296,18 +263,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
 
-func getLocalNodeIndex() int {
-	if len(globalProxyEndpoints) == 0 {
-		return -1
-	}
-	for i, ep := range globalProxyEndpoints {
-		if ep.IsLocal {
-			return i
-		}
-	}
-	return -1
-}
-
 func parseRequestToken(token string) (subToken string, nodeIndex int) {
 	if token == "" {
 		return token, -1

cmd/bucket-metadata-sys.go

@@ -24,6 +24,7 @@ import (
 	"sync"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
@@ -168,10 +169,10 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		}
 		meta.ReplicationConfigXML = configData
 	case bucketTargetsFile:
-		if !globalIsErasure && !globalIsDistErasure {
-			return NotImplemented{}
+		meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, crypto.Context{bucket: meta.Name, bucketTargetsFile: bucketTargetsFile})
+		if err != nil {
+			return fmt.Errorf("Error encrypting bucket target metadata %w", err)
 		}
-		meta.BucketTargetsConfigJSON = configData
 	default:
 		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
@@ -194,7 +195,6 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 // This function should only be used with
 // - GetBucketInfo
 // - ListBuckets
-// - ListBucketsHeal (only in case of erasure coding mode)
 // For all other bucket specific metadata, use the relevant
 // calls implemented specifically for each of those features.
 func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
@@ -443,6 +443,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	for index := range buckets {
 		index := index
 		g.Go(func() error {
+			_, _ = objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
+				// Ensure heal opts for bucket metadata be deep healed all the time.
+				ScanMode: madmin.HealDeepScan,
+			})
 			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
 			if err != nil {
 				return err
@@ -473,6 +477,15 @@ func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, ob
 	}
 }
 
+// Reset the state of the BucketMetadataSys.
+func (sys *BucketMetadataSys) Reset() {
+	sys.Lock()
+	for k := range sys.metadataMap {
+		delete(sys.metadataMap, k)
+	}
+	sys.Unlock()
+}
+
 // NewBucketMetadataSys - creates new policy system.
 func NewBucketMetadataSys() *BucketMetadataSys {
 	return &BucketMetadataSys{

cmd/bucket-metadata.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"encoding/binary"
 	"encoding/json"
 	"encoding/xml"
@@ -28,6 +29,7 @@ import (
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
@@ -37,6 +39,7 @@ import (
 	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/sio"
 )
 
 const (
@@ -61,31 +64,33 @@ var (
 // bucketMetadataFormat refers to the format.
 // bucketMetadataVersion can be used to track a rolling upgrade of a field.
 type BucketMetadata struct {
-	Name                    string
-	Created                 time.Time
-	LockEnabled             bool // legacy not used anymore.
-	PolicyConfigJSON        []byte
-	NotificationConfigXML   []byte
-	LifecycleConfigXML      []byte
-	ObjectLockConfigXML     []byte
-	VersioningConfigXML     []byte
-	EncryptionConfigXML     []byte
-	TaggingConfigXML        []byte
-	QuotaConfigJSON         []byte
-	ReplicationConfigXML    []byte
-	BucketTargetsConfigJSON []byte
+	Name                        string
+	Created                     time.Time
+	LockEnabled                 bool // legacy not used anymore.
+	PolicyConfigJSON            []byte
+	NotificationConfigXML       []byte
+	LifecycleConfigXML          []byte
+	ObjectLockConfigXML         []byte
+	VersioningConfigXML         []byte
+	EncryptionConfigXML         []byte
+	TaggingConfigXML            []byte
+	QuotaConfigJSON             []byte
+	ReplicationConfigXML        []byte
+	BucketTargetsConfigJSON     []byte
+	BucketTargetsConfigMetaJSON []byte
 
 	// Unexported fields. Must be updated atomically.
-	policyConfig       *policy.Policy
-	notificationConfig *event.Config
-	lifecycleConfig    *lifecycle.Lifecycle
-	objectLockConfig   *objectlock.Config
-	versioningConfig   *versioning.Versioning
-	sseConfig          *bucketsse.BucketSSEConfig
-	taggingConfig      *tags.Tags
-	quotaConfig        *madmin.BucketQuota
-	replicationConfig  *replication.Config
-	bucketTargetConfig *madmin.BucketTargets
+	policyConfig           *policy.Policy
+	notificationConfig     *event.Config
+	lifecycleConfig        *lifecycle.Lifecycle
+	objectLockConfig       *objectlock.Config
+	versioningConfig       *versioning.Versioning
+	sseConfig              *bucketsse.BucketSSEConfig
+	taggingConfig          *tags.Tags
+	quotaConfig            *madmin.BucketQuota
+	replicationConfig      *replication.Config
+	bucketTargetConfig     *madmin.BucketTargets
+	bucketTargetConfigMeta map[string]string
 }
 
 // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
@@ -100,7 +105,8 @@ func newBucketMetadata(name string) BucketMetadata {
 		versioningConfig: &versioning.Versioning{
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
-		bucketTargetConfig: &madmin.BucketTargets{},
+		bucketTargetConfig:     &madmin.BucketTargets{},
+		bucketTargetConfigMeta: make(map[string]string),
 	}
 }
 
@@ -140,16 +146,16 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
 func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
 	b := newBucketMetadata(bucket)
 	err := b.Load(ctx, objectAPI, b.Name)
-	if err == nil {
-		return b, b.convertLegacyConfigs(ctx, objectAPI)
-	}
-
-	if !errors.Is(err, errConfigNotFound) {
+	if err != nil && !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}
 
 	// Old bucket without bucket metadata. Hence we migrate existing settings.
-	return b, b.convertLegacyConfigs(ctx, objectAPI)
+	if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
+		return b, err
+	}
+	// migrate unencrypted remote targets
+	return b, b.migrateTargetConfig(ctx, objectAPI)
 }
 
 // parseAllConfigs will parse all configs and populate the private fields.
@@ -234,7 +240,8 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 	}
 
 	if len(b.BucketTargetsConfigJSON) != 0 {
-		if err = json.Unmarshal(b.BucketTargetsConfigJSON, b.bucketTargetConfig); err != nil {
+		b.bucketTargetConfig, err = parseBucketTargetConfig(b.Name, b.BucketTargetsConfigJSON, b.BucketTargetsConfigMetaJSON)
+		if err != nil {
 			return err
 		}
 	} else {
@@ -354,6 +361,7 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 	if err != nil {
 		return err
 	}
+
 	configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
 	return saveConfig(ctx, api, configFile, data)
 }
@@ -373,3 +381,76 @@ func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string)
 	}
 	return nil
 }
+
+// migrate config for remote targets by encrypting data if currently unencrypted and kms is configured.
+func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI ObjectLayer) error {
+	var err error
+	// early return if no targets or already encrypted
+	if len(b.BucketTargetsConfigJSON) == 0 || GlobalKMS == nil || len(b.BucketTargetsConfigMetaJSON) != 0 {
+		return nil
+	}
+
+	encBytes, metaBytes, err := encryptBucketMetadata(b.Name, b.BucketTargetsConfigJSON, crypto.Context{b.Name: b.Name, bucketTargetsFile: bucketTargetsFile})
+	if err != nil {
+		return err
+	}
+
+	b.BucketTargetsConfigJSON = encBytes
+	b.BucketTargetsConfigMetaJSON = metaBytes
+	return b.Save(ctx, objectAPI)
+}
+
+// encrypt bucket metadata if kms is configured.
+func encryptBucketMetadata(bucket string, input []byte, kmsContext crypto.Context) (output, metabytes []byte, err error) {
+	var sealedKey crypto.SealedKey
+	if GlobalKMS == nil {
+		output = input
+		return
+	}
+	var (
+		key    [32]byte
+		encKey []byte
+	)
+	metadata := make(map[string]string)
+	key, encKey, err = GlobalKMS.GenerateKey(GlobalKMS.DefaultKeyID(), kmsContext)
+	if err != nil {
+		return
+	}
+	outbuf := bytes.NewBuffer(nil)
+	objectKey := crypto.GenerateKey(key, rand.Reader)
+	sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, "")
+	crypto.S3.CreateMetadata(metadata, GlobalKMS.DefaultKeyID(), encKey, sealedKey)
+	_, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20})
+	if err != nil {
+		return output, metabytes, err
+	}
+	metabytes, err = json.Marshal(metadata)
+	if err != nil {
+		return
+	}
+	return outbuf.Bytes(), metabytes, nil
+}
+
+// decrypt bucket metadata if kms is configured.
+func decryptBucketMetadata(input []byte, bucket string, meta map[string]string, kmsContext crypto.Context) ([]byte, error) {
+	if GlobalKMS == nil {
+		return nil, errKMSNotConfigured
+	}
+	keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(meta)
+
+	if err != nil {
+		return nil, err
+	}
+	extKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, kmsContext)
+	if err != nil {
+		return nil, err
+	}
+	var objectKey crypto.ObjectKey
+	if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), bucket, ""); err != nil {
+		return nil, err
+	}
+	outbuf := bytes.NewBuffer(nil)
+	_, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20})
+
+	return outbuf.Bytes(), err
+}

cmd/bucket-metadata_gen.go

@@ -102,6 +102,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 				return
 			}
+		case "BucketTargetsConfigMetaJSON":
+			z.BucketTargetsConfigMetaJSON, err = dc.ReadBytes(z.BucketTargetsConfigMetaJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -115,9 +121,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 13
+	// map header, size 14
 	// write "Name"
-	err = en.Append(0x8d, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -246,15 +252,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 		return
 	}
+	// write "BucketTargetsConfigMetaJSON"
+	err = en.Append(0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.BucketTargetsConfigMetaJSON)
+	if err != nil {
+		err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 13
+	// map header, size 14
 	// string "Name"
-	o = append(o, 0x8d, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -292,6 +308,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "BucketTargetsConfigJSON"
 	o = append(o, 0xb7, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
 	o = msgp.AppendBytes(o, z.BucketTargetsConfigJSON)
+	// string "BucketTargetsConfigMetaJSON"
+	o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
+	o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON)
 	return
 }
 
@@ -391,6 +410,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 				return
 			}
+		case "BucketTargetsConfigMetaJSON":
+			z.BucketTargetsConfigMetaJSON, bts, err = msgp.ReadBytesBytes(bts, z.BucketTargetsConfigMetaJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -405,6 +430,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON)
 	return
 }

cmd/bucket-object-lock.go

@@ -21,10 +21,12 @@ import (
 	"math"
 	"net/http"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/replication"
 )
 
 // BucketObjectLockSys - map of bucket and retention configuration.
@@ -80,24 +82,25 @@ func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locke
 // For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
 // governance bypass headers are set and user has governance bypass permissions.
 // Objects in "Compliance" mode can be overwritten only if retention date is past.
-func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, getObjectInfoFn GetObjectInfoFn) APIErrorCode {
+func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, oi ObjectInfo, gerr error) APIErrorCode {
 	opts, err := getOpts(ctx, r, bucket, object.ObjectName)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}
 
 	opts.VersionID = object.VersionID
-
-	oi, err := getObjectInfoFn(ctx, bucket, object.ObjectName, opts)
-	if err != nil {
-		switch err.(type) {
+	if gerr != nil { // error from GetObjectInfo
+		switch gerr.(type) {
 		case MethodNotAllowed: // This happens usually for a delete marker
 			if oi.DeleteMarker {
 				// Delete marker should be present and valid.
 				return ErrNone
 			}
 		}
-		return toAPIErrorCode(ctx, err)
+		if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) {
+			return ErrNone
+		}
+		return toAPIErrorCode(ctx, gerr)
 	}
 
 	lhold := objectlock.GetObjectLegalHoldMeta(oi.UserDefined)
@@ -245,13 +248,13 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
 // For objects in "Compliance" mode, retention date cannot be shortened, and mode cannot be altered.
 // For objects with legal hold header set, the s3:PutObjectLegalHold permission is expected to be set
 // Both legal hold and retention can be applied independently on an object
-func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
+func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
 	var mode objectlock.RetMode
 	var retainDate objectlock.RetentionDate
 	var legalHold objectlock.ObjectLegalHold
 
-	retentionRequested := objectlock.IsObjectLockRetentionRequested(r.Header)
-	legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(r.Header)
+	retentionRequested := objectlock.IsObjectLockRetentionRequested(rq.Header)
+	legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(rq.Header)
 
 	retentionCfg, err := globalBucketObjectLockSys.Get(bucket)
 	if err != nil {
@@ -267,25 +270,24 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		return mode, retainDate, legalHold, ErrNone
 	}
 
-	opts, err := getOpts(ctx, r, bucket, object)
+	opts, err := getOpts(ctx, rq, bucket, object)
 	if err != nil {
 		return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 	}
 
-	if opts.VersionID != "" {
+	replica := rq.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String()
+
+	if opts.VersionID != "" && !replica {
 		if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
 			r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
-
 			t, err := objectlock.UTCNowNTP()
 			if err != nil {
 				logger.LogIf(ctx, err)
 				return mode, retainDate, legalHold, ErrObjectLocked
 			}
-
 			if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
 				return mode, retainDate, legalHold, ErrObjectLocked
 			}
-
 			mode = r.Mode
 			retainDate = r.RetainUntilDate
 			legalHold = objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
@@ -298,17 +300,17 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 
 	if legalHoldRequested {
 		var lerr error
-		if legalHold, lerr = objectlock.ParseObjectLockLegalHoldHeaders(r.Header); lerr != nil {
+		if legalHold, lerr = objectlock.ParseObjectLockLegalHoldHeaders(rq.Header); lerr != nil {
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
 	}
 
 	if retentionRequested {
-		legalHold, err := objectlock.ParseObjectLockLegalHoldHeaders(r.Header)
+		legalHold, err := objectlock.ParseObjectLockLegalHoldHeaders(rq.Header)
 		if err != nil {
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
-		rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(r.Header)
+		rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header)
 		if err != nil {
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
@@ -317,7 +319,9 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		}
 		return rMode, rDate, legalHold, ErrNone
 	}
-
+	if replica { // replica inherits retention metadata only from source
+		return "", objectlock.RetentionDate{}, legalHold, ErrNone
+	}
 	if !retentionRequested && retentionCfg.Validity > 0 {
 		if retentionPermErr != ErrNone {
 			return mode, retainDate, legalHold, retentionPermErr

cmd/bucket-replication.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 	"time"
 
+	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -34,6 +35,7 @@ import (
 	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/event"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // gets replication config associated to a given bucket name.
@@ -53,6 +55,13 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
 // It also returns true if replication destination is same as this server.
 func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, error) {
+	arn, err := madmin.ParseARN(rCfg.RoleArn)
+	if err != nil {
+		return false, BucketRemoteArnInvalid{}
+	}
+	if arn.Type != madmin.ReplicationService {
+		return false, BucketRemoteArnTypeInvalid{}
+	}
 	clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rCfg.RoleArn)
 	if clnt == nil {
 		return false, BucketRemoteTargetNotFound{Bucket: bucket}
@@ -79,35 +88,37 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 	return false, BucketRemoteTargetNotFound{Bucket: bucket}
 }
 
-func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) bool {
+func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) (replicate bool, sync bool) {
 	if permErr != ErrNone {
-		return false
+		return
 	}
-	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+	return mustReplicater(ctx, bucket, object, meta, replStatus)
 }
 
-// mustReplicate returns true if object meets replication criteria.
-func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
-	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
-		return false
+// mustReplicate returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
+// a synchronous manner.
+func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) (replicate bool, sync bool) {
+	if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
+		return
 	}
-	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+	return mustReplicater(ctx, bucket, object, meta, replStatus)
 }
 
-// mustReplicater returns true if object meets replication criteria.
-func mustReplicater(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
+// mustReplicater returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
+// a synchronous manner.
+func mustReplicater(ctx context.Context, bucket, object string, meta map[string]string, replStatus string) (replicate bool, sync bool) {
 	if globalIsGateway {
-		return false
+		return replicate, sync
 	}
 	if rs, ok := meta[xhttp.AmzBucketReplicationStatus]; ok {
 		replStatus = rs
 	}
 	if replication.StatusType(replStatus) == replication.Replica {
-		return false
+		return replicate, sync
 	}
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
-		return false
+		return replicate, sync
 	}
 	opts := replication.ObjectOpts{
 		Name: object,
@@ -117,16 +128,208 @@ func mustReplicater(ctx context.Context, r *http.Request, bucket, object string,
 	if ok {
 		opts.UserTags = tagStr
 	}
-	return cfg.Replicate(opts)
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
+	if tgt == nil || tgt.isOffline() {
+		return cfg.Replicate(opts), false
+	}
+	return cfg.Replicate(opts), tgt.replicateSync
+}
+
+// Standard headers that needs to be extracted from User metadata.
+var standardHeaders = []string{
+	"content-type",
+	"content-encoding",
+	xhttp.AmzStorageClass,
+	xhttp.AmzObjectTagging,
+	xhttp.AmzBucketReplicationStatus,
+}
+
+// returns true if any of the objects being deleted qualifies for replication.
+func hasReplicationRules(ctx context.Context, bucket string, objects []ObjectToDelete) bool {
+	c, err := getReplicationConfig(ctx, bucket)
+	if err != nil || c == nil {
+		return false
+	}
+	for _, obj := range objects {
+		if c.HasActiveRules(obj.ObjectName, true) {
+			return true
+		}
+	}
+	return false
+}
+
+// isStandardHeader returns true if header is a supported header and not a custom header
+func isStandardHeader(headerKey string) bool {
+	key := strings.ToLower(headerKey)
+	for _, header := range standardHeaders {
+		if strings.ToLower(header) == key {
+			return true
+		}
+	}
+	return false
+}
+
+// returns whether object version is a deletemarker and if object qualifies for replication
+func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelete, oi ObjectInfo, gerr error) (dm, replicate, sync bool) {
+	rcfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil || rcfg == nil {
+		return false, false, sync
+	}
+	// when incoming delete is removal of a delete marker( a.k.a versioned delete),
+	// GetObjectInfo returns extra information even though it returns errFileNotFound
+	if gerr != nil {
+		validReplStatus := false
+		switch oi.ReplicationStatus {
+		case replication.Pending, replication.Completed, replication.Failed:
+			validReplStatus = true
+		}
+		if oi.DeleteMarker && validReplStatus {
+			return oi.DeleteMarker, true, sync
+		}
+		return oi.DeleteMarker, false, sync
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
+	if tgt == nil || tgt.isOffline() {
+		return oi.DeleteMarker, false, false
+	}
+	opts := replication.ObjectOpts{
+		Name:         dobj.ObjectName,
+		SSEC:         crypto.SSEC.IsEncrypted(oi.UserDefined),
+		UserTags:     oi.UserTags,
+		DeleteMarker: oi.DeleteMarker,
+		VersionID:    dobj.VersionID,
+	}
+	return oi.DeleteMarker, rcfg.Replicate(opts), tgt.replicateSync
+}
+
+// replicate deletes to the designated replication target if replication configuration
+// has delete marker replication or delete replication (MinIO extension to allow deletes where version id
+// is specified) enabled.
+// Similar to bucket replication for PUT operation, soft delete (a.k.a setting delete marker) and
+// permanent deletes (by specifying a version ID in the delete operation) have three states "Pending", "Complete"
+// and "Failed" to mark the status of the replication of "DELETE" operation. All failed operations can
+// then be retried by healing. In the case of permanent deletes, until the replication is completed on the
+// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
+// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
+// on target.
+func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectAPI ObjectLayer) {
+	bucket := dobj.Bucket
+	rcfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil || rcfg == nil {
+		return
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
+	if tgt == nil {
+		return
+	}
+	versionID := dobj.DeleteMarkerVersionID
+	if versionID == "" {
+		versionID = dobj.VersionID
+	}
+	rmErr := tgt.RemoveObject(ctx, rcfg.GetDestination().Bucket, dobj.ObjectName, miniogo.RemoveObjectOptions{
+		VersionID: versionID,
+		Internal: miniogo.AdvancedRemoveOptions{
+			ReplicationDeleteMarker: dobj.DeleteMarkerVersionID != "",
+			ReplicationMTime:        dobj.DeleteMarkerMTime.Time,
+			ReplicationStatus:       miniogo.ReplicationStatusReplica,
+		},
+	})
+
+	replicationStatus := dobj.DeleteMarkerReplicationStatus
+	versionPurgeStatus := dobj.VersionPurgeStatus
+
+	if rmErr != nil {
+		if dobj.VersionID == "" {
+			replicationStatus = string(replication.Failed)
+		} else {
+			versionPurgeStatus = Failed
+		}
+	} else {
+		if dobj.VersionID == "" {
+			replicationStatus = string(replication.Completed)
+		} else {
+			versionPurgeStatus = Complete
+		}
+	}
+	var eventName = event.ObjectReplicationComplete
+	if replicationStatus == string(replication.Failed) || versionPurgeStatus == Failed {
+		eventName = event.ObjectReplicationFailed
+	}
+	objInfo := ObjectInfo{
+		Name:               dobj.ObjectName,
+		DeleteMarker:       dobj.DeleteMarker,
+		VersionID:          versionID,
+		ReplicationStatus:  replication.StatusType(dobj.DeleteMarkerReplicationStatus),
+		VersionPurgeStatus: versionPurgeStatus,
+	}
+
+	eventArg := &eventArgs{
+		BucketName: bucket,
+		Object:     objInfo,
+		Host:       "Internal: [Replication]",
+		EventName:  eventName,
+	}
+	sendEvent(*eventArg)
+
+	// Update metadata on the delete marker or purge permanent delete if replication success.
+	if _, err = objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{
+		VersionID:                     versionID,
+		DeleteMarker:                  dobj.DeleteMarker,
+		DeleteMarkerReplicationStatus: replicationStatus,
+		Versioned:                     globalBucketVersioningSys.Enabled(bucket),
+		VersionPurgeStatus:            versionPurgeStatus,
+		VersionSuspended:              globalBucketVersioningSys.Suspended(bucket),
+	}); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s %s: %w", bucket, dobj.ObjectName, dobj.VersionID, err))
+	}
+}
+
+func getCopyObjMetadata(oi ObjectInfo, dest replication.Destination) map[string]string {
+	meta := make(map[string]string, len(oi.UserDefined))
+	for k, v := range oi.UserDefined {
+		if k == xhttp.AmzBucketReplicationStatus {
+			continue
+		}
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			continue
+		}
+		meta[k] = v
+	}
+	if oi.ContentEncoding != "" {
+		meta[xhttp.ContentEncoding] = oi.ContentEncoding
+	}
+	if oi.ContentType != "" {
+		meta[xhttp.ContentType] = oi.ContentType
+	}
+	tag, err := tags.ParseObjectTags(oi.UserTags)
+	if err != nil {
+		return nil
+	}
+	if tag != nil {
+		meta[xhttp.AmzObjectTagging] = tag.String()
+		meta[xhttp.AmzTagDirective] = "REPLACE"
+	}
+	sc := dest.StorageClass
+	if sc == "" {
+		sc = oi.StorageClass
+	}
+	meta[xhttp.AmzStorageClass] = sc
+	if oi.UserTags != "" {
+		meta[xhttp.AmzObjectTagging] = oi.UserTags
+	}
+	meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339Nano)
+	meta[xhttp.MinIOSourceETag] = oi.ETag
+	meta[xhttp.AmzBucketReplicationStatus] = replication.Replica.String()
+	return meta
 }
 
 func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
-		if k == xhttp.AmzBucketReplicationStatus {
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			continue
 		}
-		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+		if isStandardHeader(k) {
 			continue
 		}
 		meta[k] = v
@@ -157,7 +360,7 @@ func putReplicationOpts(ctx context.Context, dest replication.Destination, objIn
 		putOpts.Mode = rmode
 	}
 	if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
-		rdate, err := time.Parse(time.RFC3339, retainDateStr)
+		rdate, err := time.Parse(time.RFC3339Nano, retainDateStr)
 		if err != nil {
 			return
 		}
@@ -173,6 +376,53 @@ func putReplicationOpts(ctx context.Context, dest replication.Destination, objIn
 	return
 }
 
+type replicationAction string
+
+const (
+	replicateMetadata replicationAction = "metadata"
+	replicateNone     replicationAction = "none"
+	replicateAll      replicationAction = "all"
+)
+
+// returns replicationAction by comparing metadata between source and target
+func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo) replicationAction {
+	// needs full replication
+	if oi1.ETag != oi2.ETag ||
+		oi1.VersionID != oi2.VersionID ||
+		oi1.Size != oi2.Size ||
+		oi1.DeleteMarker != oi2.IsDeleteMarker {
+		return replicateAll
+	}
+
+	if !oi1.ModTime.Equal(oi2.LastModified) ||
+		oi1.ContentType != oi2.ContentType ||
+		oi1.StorageClass != oi2.StorageClass {
+		return replicateMetadata
+	}
+	if oi1.ContentEncoding != "" {
+		enc, ok := oi2.UserMetadata[xhttp.ContentEncoding]
+		if !ok || enc != oi1.ContentEncoding {
+			return replicateMetadata
+		}
+	}
+	for k, v := range oi2.UserMetadata {
+		oi2.Metadata[k] = []string{v}
+	}
+	if len(oi2.Metadata) != len(oi1.UserDefined) {
+		return replicateMetadata
+	}
+	for k1, v1 := range oi1.UserDefined {
+		if v2, ok := oi2.Metadata[k1]; !ok || v1 != strings.Join(v2, "") {
+			return replicateMetadata
+		}
+	}
+	t, _ := tags.MapToObjectTags(oi2.UserTags)
+	if t.String() != oi1.UserTags {
+		return replicateMetadata
+	}
+	return replicateNone
+}
+
 // replicateObject replicates the specified version of the object to destination bucket
 // The source object is then updated to reflect the replication status.
 func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLayer) {
@@ -193,6 +443,7 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 		VersionID: objInfo.VersionID,
 	})
 	if err != nil {
+		logger.LogIf(ctx, err)
 		return
 	}
 	objInfo = gr.ObjInfo
@@ -209,44 +460,56 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 		return
 	}
 
-	// if heal encounters a pending replication status, either replication
-	// has failed due to server shutdown or crawler and PutObject replication are in contention.
-	healPending := objInfo.ReplicationStatus == replication.Pending
-
-	// In the rare event that replication is in pending state either due to
-	// server shut down/crash before replication completed or healing and PutObject
-	// race - do an additional stat to see if the version ID exists
-	if healPending {
-		_, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID})
-		if err == nil {
+	rtype := replicateAll
+	oi, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID})
+	if err == nil {
+		rtype = getReplicationAction(objInfo, oi)
+		if rtype == replicateNone {
 			gr.Close()
 			// object with same VersionID already exists, replication kicked off by
 			// PutObject might have completed.
 			return
 		}
 	}
+	replicationStatus := replication.Completed
+	if rtype != replicateAll {
+		gr.Close()
 
-	target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
-		return
-	}
-	putOpts := putReplicationOpts(ctx, dest, objInfo)
-	replicationStatus := replication.Complete
+		// replicate metadata for object tagging/copy with metadata replacement
+		dstOpts := miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{SourceVersionID: objInfo.VersionID}}
+		c := &miniogo.Core{Client: tgt.Client}
+		_, err = c.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), dstOpts)
+		if err != nil {
+			replicationStatus = replication.Failed
+		}
+	} else {
+		target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
+			gr.Close()
+			return
+		}
 
-	// Setup bandwidth throttling
-	totalNodesCount := len(GetRemotePeers(globalEndpoints)) + 1
-	b := target.BandwidthLimit / int64(totalNodesCount)
-	var headerSize int
-	for k, v := range putOpts.Header() {
-		headerSize += len(k) + len(v)
-	}
-	r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
+		putOpts := putReplicationOpts(ctx, dest, objInfo)
+		// Setup bandwidth throttling
+		peers, _ := globalEndpoints.peers()
+		totalNodesCount := len(peers)
+		if totalNodesCount == 0 {
+			totalNodesCount = 1 // For standalone erasure coding
+		}
+		b := target.BandwidthLimit / int64(totalNodesCount)
+		var headerSize int
+		for k, v := range putOpts.Header() {
+			headerSize += len(k) + len(v)
+		}
 
-	_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts)
-	r.Close()
-	if err != nil {
-		replicationStatus = replication.Failed
+		// r takes over closing gr.
+		r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
+		_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, putOpts)
+		if err != nil {
+			replicationStatus = replication.Failed
+		}
+		r.Close()
 	}
 	objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
 	if objInfo.UserTags != "" {
@@ -257,15 +520,16 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 	// - event.ObjectReplicationNotTracked
 	// - event.ObjectReplicationMissedThreshold
 	// - event.ObjectReplicationReplicatedAfterThreshold
+	var eventName = event.ObjectReplicationComplete
 	if replicationStatus == replication.Failed {
-		sendEvent(eventArgs{
-			EventName:  event.ObjectReplicationFailed,
-			BucketName: bucket,
-			Object:     objInfo,
-			Host:       "Internal: [Replication]",
-		})
+		eventName = event.ObjectReplicationFailed
 	}
-
+	sendEvent(eventArgs{
+		EventName:  eventName,
+		BucketName: bucket,
+		Object:     objInfo,
+		Host:       "Internal: [Replication]",
+	})
 	objInfo.metadataOnly = true // Perform only metadata updates.
 	if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{
 		VersionID: objInfo.VersionID,
@@ -299,18 +563,37 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
 	return dst
 }
 
+// DeletedObjectVersionInfo has info on deleted object
+type DeletedObjectVersionInfo struct {
+	DeletedObject
+	Bucket string
+}
 type replicationState struct {
 	// add future metrics here
-	replicaCh chan ObjectInfo
+	replicaCh       chan ObjectInfo
+	replicaDeleteCh chan DeletedObjectVersionInfo
 }
 
 func (r *replicationState) queueReplicaTask(oi ObjectInfo) {
+	if r == nil {
+		return
+	}
 	select {
 	case r.replicaCh <- oi:
 	default:
 	}
 }
 
+func (r *replicationState) queueReplicaDeleteTask(doi DeletedObjectVersionInfo) {
+	if r == nil {
+		return
+	}
+	select {
+	case r.replicaDeleteCh <- doi:
+	default:
+	}
+}
+
 var (
 	globalReplicationState *replicationState
 	// TODO: currently keeping it conservative
@@ -327,11 +610,13 @@ func newReplicationState() *replicationState {
 		globalReplicationConcurrent = 1
 	}
 	rs := &replicationState{
-		replicaCh: make(chan ObjectInfo, 10000),
+		replicaCh:       make(chan ObjectInfo, 10000),
+		replicaDeleteCh: make(chan DeletedObjectVersionInfo, 10000),
 	}
 	go func() {
 		<-GlobalContext.Done()
 		close(rs.replicaCh)
+		close(rs.replicaDeleteCh)
 	}()
 	return rs
 }
@@ -349,6 +634,11 @@ func (r *replicationState) addWorker(ctx context.Context, objectAPI ObjectLayer)
 					return
 				}
 				replicateObject(ctx, oi, objectAPI)
+			case doi, ok := <-r.replicaDeleteCh:
+				if !ok {
+					return
+				}
+				replicateDelete(ctx, doi, objectAPI)
 			}
 		}
 	}()
@@ -364,3 +654,140 @@ func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
 		globalReplicationState.addWorker(ctx, objectAPI)
 	}
 }
+
+// get Reader from replication target if active-active replication is in place and
+// this node returns a 404
+func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, proxy bool) {
+	tgt, oi, proxy, err := proxyHeadToRepTarget(ctx, bucket, object, opts)
+	if !proxy || err != nil {
+		return nil, false
+	}
+	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
+	if err != nil {
+		return nil, false
+	}
+	gopts := miniogo.GetObjectOptions{
+		VersionID:            opts.VersionID,
+		ServerSideEncryption: opts.ServerSideEncryption,
+		Internal: miniogo.AdvancedGetOptions{
+			ReplicationProxyRequest: true,
+		},
+	}
+	// get correct offsets for encrypted object
+	if off >= 0 && length >= 0 {
+		if err := gopts.SetRange(off, off+length-1); err != nil {
+			return nil, false
+		}
+	}
+	c := miniogo.Core{Client: tgt.Client}
+
+	obj, _, _, err := c.GetObject(ctx, bucket, object, gopts)
+	if err != nil {
+		return nil, false
+	}
+	closeReader := func() { obj.Close() }
+
+	reader, err := fn(obj, h, opts.CheckPrecondFn, closeReader)
+	if err != nil {
+		return nil, false
+	}
+	return reader, true
+}
+
+// isProxyable returns true if replication config found for this bucket
+func isProxyable(ctx context.Context, bucket string) bool {
+	cfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		return false
+	}
+	dest := cfg.GetDestination()
+	return dest.Bucket == bucket
+}
+func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions) (tgt *TargetClient, oi ObjectInfo, proxy bool, err error) {
+	// this option is set when active-active replication is in place between site A -> B,
+	// and site B does not have the object yet.
+	if opts.ProxyRequest { // true only when site B sets MinIOSourceProxyRequest header
+		return nil, oi, false, nil
+	}
+	cfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		return nil, oi, false, err
+	}
+	dest := cfg.GetDestination()
+	if dest.Bucket != bucket { // not active-active
+		return nil, oi, false, err
+	}
+	ssec := false
+	if opts.ServerSideEncryption != nil {
+		ssec = opts.ServerSideEncryption.Type() == encrypt.SSEC
+	}
+	ropts := replication.ObjectOpts{
+		Name: object,
+		SSEC: ssec,
+	}
+	if !cfg.Replicate(ropts) { // no matching rule for object prefix
+		return nil, oi, false, nil
+	}
+	tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
+	if tgt == nil || tgt.isOffline() {
+		return nil, oi, false, fmt.Errorf("missing target")
+	}
+
+	gopts := miniogo.GetObjectOptions{
+		VersionID:            opts.VersionID,
+		ServerSideEncryption: opts.ServerSideEncryption,
+		Internal: miniogo.AdvancedGetOptions{
+			ReplicationProxyRequest: true,
+		},
+	}
+
+	objInfo, err := tgt.StatObject(ctx, dest.Bucket, object, gopts)
+	if err != nil {
+		return nil, oi, false, err
+	}
+	tags, _ := tags.MapToObjectTags(objInfo.UserTags)
+	oi = ObjectInfo{
+		Bucket:            bucket,
+		Name:              object,
+		ModTime:           objInfo.LastModified,
+		Size:              objInfo.Size,
+		ETag:              objInfo.ETag,
+		VersionID:         objInfo.VersionID,
+		IsLatest:          objInfo.IsLatest,
+		DeleteMarker:      objInfo.IsDeleteMarker,
+		ContentType:       objInfo.ContentType,
+		Expires:           objInfo.Expires,
+		StorageClass:      objInfo.StorageClass,
+		ReplicationStatus: replication.StatusType(objInfo.ReplicationStatus),
+		UserDefined:       cloneMSS(objInfo.UserMetadata),
+		UserTags:          tags.String(),
+	}
+	if ce, ok := oi.UserDefined[xhttp.ContentEncoding]; ok {
+		oi.ContentEncoding = ce
+		delete(oi.UserDefined, xhttp.ContentEncoding)
+	}
+	return tgt, oi, true, nil
+}
+
+// get object info from replication target if active-active replication is in place and
+// this node returns a 404
+func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, proxy bool, err error) {
+	_, oi, proxy, err = proxyHeadToRepTarget(ctx, bucket, object, opts)
+	return oi, proxy, err
+}
+
+func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer, sync bool) {
+	if sync {
+		replicateObject(ctx, objInfo, o)
+	} else {
+		globalReplicationState.queueReplicaTask(objInfo)
+	}
+}
+
+func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectVersionInfo, o ObjectLayer, sync bool) {
+	if sync {
+		replicateDelete(ctx, dv, o)
+	} else {
+		globalReplicationState.queueReplicaDeleteTask(dv)
+	}
+}

cmd/bucket-targets.go

@@ -18,23 +18,32 @@ package cmd
 
 import (
 	"context"
+	"encoding/hex"
+	"encoding/json"
 	"net/http"
+	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/madmin"
+	sha256 "github.com/minio/sha256-simd"
+)
+
+const (
+	defaultHealthCheckDuration = 60 * time.Second
 )
 
 // BucketTargetSys represents bucket targets subsystem
 type BucketTargetSys struct {
 	sync.RWMutex
-	arnRemotesMap map[string]*miniogo.Core
+	arnRemotesMap map[string]*TargetClient
 	targetsMap    map[string][]madmin.BucketTarget
-	clientsCache  map[string]*miniogo.Core
 }
 
 // ListTargets lists bucket targets across tenant or for individual bucket, and returns
@@ -64,7 +73,6 @@ func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType str
 
 // ListBucketTargets - gets list of bucket targets for this bucket.
 func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
-
 	sys.RLock()
 	defer sys.RUnlock()
 
@@ -76,44 +84,67 @@ func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string
 }
 
 // SetTarget - sets a new minio-go client target for this bucket.
-func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget) error {
+func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget, update bool) error {
 	if globalIsGateway {
 		return nil
 	}
-	if !tgt.Type.IsValid() {
+	if !tgt.Type.IsValid() && !update {
 		return BucketRemoteArnTypeInvalid{Bucket: bucket}
 	}
 	clnt, err := sys.getRemoteTargetClient(tgt)
 	if err != nil {
 		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 	}
+	// validate if target credentials are ok
+	if _, err = clnt.BucketExists(ctx, tgt.TargetBucket); err != nil {
+		if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
+			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+		}
+		return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
+	}
 	if tgt.Type == madmin.ReplicationService {
+		if !globalIsErasure {
+			return NotImplemented{}
+		}
 		if !globalBucketVersioningSys.Enabled(bucket) {
 			return BucketReplicationSourceNotVersioned{Bucket: bucket}
 		}
 		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
 		if err != nil {
-			if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
-				return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
-			}
 			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
 		}
 		if vcfg.Status != string(versioning.Enabled) {
 			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
 		}
 	}
+	if tgt.Type == madmin.ILMService {
+		if globalBucketVersioningSys.Enabled(bucket) {
+			vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
+			if err != nil {
+				if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
+					return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+				}
+				return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
+			}
+			if vcfg.Status != string(versioning.Enabled) {
+				return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
+			}
+		}
+	}
 	sys.Lock()
 	defer sys.Unlock()
 
 	tgts := sys.targetsMap[bucket]
 	newtgts := make([]madmin.BucketTarget, len(tgts))
+	labels := make(map[string]struct{}, len(tgts))
 	found := false
 	for idx, t := range tgts {
+		labels[t.Label] = struct{}{}
 		if t.Type == tgt.Type {
-			if t.Arn == tgt.Arn {
+			if t.Arn == tgt.Arn && !update {
 				return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
 			}
-			if t.Label == tgt.Label {
+			if t.Label == tgt.Label && !update {
 				return BucketRemoteLabelInUse{Bucket: t.TargetBucket}
 			}
 			newtgts[idx] = *tgt
@@ -122,15 +153,15 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		}
 		newtgts[idx] = t
 	}
-	if !found {
+	if _, ok := labels[tgt.Label]; ok && !update {
+		return BucketRemoteLabelInUse{Bucket: tgt.TargetBucket}
+	}
+	if !found && !update {
 		newtgts = append(newtgts, *tgt)
 	}
 
 	sys.targetsMap[bucket] = newtgts
 	sys.arnRemotesMap[tgt.Arn] = clnt
-	if _, ok := sys.clientsCache[clnt.EndpointURL().String()]; !ok {
-		sys.clientsCache[clnt.EndpointURL().String()] = clnt
-	}
 	return nil
 }
 
@@ -147,6 +178,9 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 		return BucketRemoteArnInvalid{Bucket: bucket}
 	}
 	if arn.Type == madmin.ReplicationService {
+		if !globalIsErasure {
+			return NotImplemented{}
+		}
 		// reject removal of remote target if replication configuration is present
 		rcfg, err := getReplicationConfig(ctx, bucket)
 		if err == nil && rcfg.RoleArn == arnStr {
@@ -155,12 +189,25 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 			}
 		}
 	}
+	if arn.Type == madmin.ILMService {
+		// reject removal of remote target if lifecycle transition uses this arn
+		config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+		if err == nil && transitionSCInUse(ctx, config, bucket, arnStr) {
+			if _, ok := sys.arnRemotesMap[arnStr]; ok {
+				return BucketRemoteRemoveDisallowed{Bucket: bucket}
+			}
+		}
+	}
+
 	// delete ARN type from list of matching targets
 	sys.Lock()
 	defer sys.Unlock()
-	targets := make([]madmin.BucketTarget, 0)
 	found := false
-	tgts := sys.targetsMap[bucket]
+	tgts, ok := sys.targetsMap[bucket]
+	if !ok {
+		return BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	targets := make([]madmin.BucketTarget, 0, len(tgts))
 	for _, tgt := range tgts {
 		if tgt.Arn != arnStr {
 			targets = append(targets, tgt)
@@ -177,18 +224,55 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 }
 
 // GetRemoteTargetClient returns minio-go client for replication target instance
-func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *miniogo.Core {
+func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *TargetClient {
 	sys.RLock()
 	defer sys.RUnlock()
 	return sys.arnRemotesMap[arn]
 }
 
+// GetRemoteTargetWithLabel returns bucket target given a target label
+func (sys *BucketTargetSys) GetRemoteTargetWithLabel(ctx context.Context, bucket, targetLabel string) *madmin.BucketTarget {
+	sys.RLock()
+	defer sys.RUnlock()
+	for _, t := range sys.targetsMap[bucket] {
+		if strings.ToUpper(t.Label) == strings.ToUpper(targetLabel) {
+			tgt := t.Clone()
+			return &tgt
+		}
+	}
+	return nil
+}
+
+// GetRemoteArnWithLabel returns bucket target's ARN given its target label
+func (sys *BucketTargetSys) GetRemoteArnWithLabel(ctx context.Context, bucket, tgtLabel string) *madmin.ARN {
+	tgt := sys.GetRemoteTargetWithLabel(ctx, bucket, tgtLabel)
+	if tgt == nil {
+		return nil
+	}
+	arn, err := madmin.ParseARN(tgt.Arn)
+	if err != nil {
+		return nil
+	}
+	return arn
+}
+
+// GetRemoteLabelWithArn returns a bucket target's label given its ARN
+func (sys *BucketTargetSys) GetRemoteLabelWithArn(ctx context.Context, bucket, arnStr string) string {
+	sys.RLock()
+	defer sys.RUnlock()
+	for _, t := range sys.targetsMap[bucket] {
+		if t.Arn == arnStr {
+			return t.Label
+		}
+	}
+	return ""
+}
+
 // NewBucketTargetSys - creates new replication system.
 func NewBucketTargetSys() *BucketTargetSys {
 	return &BucketTargetSys{
-		arnRemotesMap: make(map[string]*miniogo.Core),
+		arnRemotesMap: make(map[string]*TargetClient),
 		targetsMap:    make(map[string][]madmin.BucketTarget),
-		clientsCache:  make(map[string]*miniogo.Core),
 	}
 }
 
@@ -235,9 +319,6 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 			continue
 		}
 		sys.arnRemotesMap[tgt.Arn] = tgtClient
-		if _, ok := sys.clientsCache[tgtClient.EndpointURL().String()]; !ok {
-			sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
-		}
 	}
 	sys.targetsMap[bucket] = tgts.Targets
 }
@@ -261,9 +342,6 @@ func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objA
 				continue
 			}
 			sys.arnRemotesMap[tgt.Arn] = tgtClient
-			if _, ok := sys.clientsCache[tgtClient.EndpointURL().String()]; !ok {
-				sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
-			}
 		}
 		sys.targetsMap[bucket.Name] = cfg.Targets
 	}
@@ -274,23 +352,33 @@ var getRemoteTargetInstanceTransport http.RoundTripper
 var getRemoteTargetInstanceTransportOnce sync.Once
 
 // Returns a minio-go Client configured to access remote host described in replication target config.
-func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*miniogo.Core, error) {
-	if clnt, ok := sys.clientsCache[tcfg.Endpoint]; ok {
-		return clnt, nil
-	}
+func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*TargetClient, error) {
 	config := tcfg.Credentials
 	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
 
 	getRemoteTargetInstanceTransportOnce.Do(func() {
-		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(1 * time.Hour)
+		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
 	})
-
-	core, err := miniogo.NewCore(tcfg.Endpoint, &miniogo.Options{
+	api, err := minio.New(tcfg.Endpoint, &miniogo.Options{
 		Creds:     creds,
 		Secure:    tcfg.Secure,
 		Transport: getRemoteTargetInstanceTransport,
 	})
-	return core, err
+	if err != nil {
+		return nil, err
+	}
+	hcDuration := defaultHealthCheckDuration
+	if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
+		hcDuration = tcfg.HealthCheckDuration
+	}
+	tc := &TargetClient{
+		Client:              api,
+		healthCheckDuration: hcDuration,
+		bucket:              tcfg.TargetBucket,
+		replicateSync:       tcfg.ReplicationSync,
+	}
+	go tc.healthCheck()
+	return tc, nil
 }
 
 // getRemoteARN gets existing ARN for an endpoint or generates a new one.
@@ -300,18 +388,83 @@ func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTar
 	}
 	tgts := sys.targetsMap[bucket]
 	for _, tgt := range tgts {
-		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL() == tgt.URL() {
+		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL().String() == tgt.URL().String() {
 			return tgt.Arn
 		}
 	}
 	if !madmin.ServiceType(target.Type).IsValid() {
 		return ""
 	}
+	return generateARN(target)
+}
+
+// generate ARN that is unique to this target type
+func generateARN(t *madmin.BucketTarget) string {
+	hash := sha256.New()
+	hash.Write([]byte(t.Type))
+	hash.Write([]byte(t.Region))
+	hash.Write([]byte(t.TargetBucket))
+	hashSum := hex.EncodeToString(hash.Sum(nil))
 	arn := madmin.ARN{
-		Type:   target.Type,
-		ID:     mustGetUUID(),
-		Region: target.Region,
-		Bucket: target.TargetBucket,
+		Type:   t.Type,
+		ID:     hashSum,
+		Region: t.Region,
+		Bucket: t.TargetBucket,
 	}
 	return arn.String()
 }
+
+// Returns parsed target config. If KMS is configured, remote target is decrypted
+func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.BucketTargets, error) {
+	var (
+		data []byte
+		err  error
+		t    madmin.BucketTargets
+		meta map[string]string
+	)
+	if len(cdata) == 0 {
+		return nil, nil
+	}
+	data = cdata
+	if len(cmetadata) != 0 {
+		if err := json.Unmarshal(cmetadata, &meta); err != nil {
+			return nil, err
+		}
+		if crypto.S3.IsEncrypted(meta) {
+			if data, err = decryptBucketMetadata(cdata, bucket, meta, crypto.Context{bucket: bucket, bucketTargetsFile: bucketTargetsFile}); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	if err = json.Unmarshal(data, &t); err != nil {
+		return nil, err
+	}
+	return &t, nil
+}
+
+// TargetClient is the struct for remote target client.
+type TargetClient struct {
+	*miniogo.Client
+	up                  int32
+	healthCheckDuration time.Duration
+	bucket              string // remote bucket target
+	replicateSync       bool
+}
+
+func (tc *TargetClient) isOffline() bool {
+	return atomic.LoadInt32(&tc.up) == 0
+}
+
+func (tc *TargetClient) healthCheck() {
+	for {
+		_, err := tc.BucketExists(GlobalContext, tc.bucket)
+		if err != nil {
+			atomic.StoreInt32(&tc.up, 0)
+			time.Sleep(tc.healthCheckDuration * time.Second)
+			continue
+		}
+		atomic.StoreInt32(&tc.up, 1)
+		time.Sleep(tc.healthCheckDuration * time.Second)
+	}
+}

cmd/bucket-versioning.go

@@ -51,6 +51,11 @@ func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, erro
 	return globalBucketMetadataSys.GetVersioningConfig(bucket)
 }
 
+// Reset BucketVersioningSys to initial state.
+func (sys *BucketVersioningSys) Reset() {
+	// There is currently no internal state.
+}
+
 // NewBucketVersioningSys - creates new versioning system.
 func NewBucketVersioningSys() *BucketVersioningSys {
 	return &BucketVersioningSys{}

cmd/common-main.go

@@ -30,6 +30,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/fatih/color"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -38,15 +39,26 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/env"
 )
 
+// serverDebugLog will enable debug printing
+var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+
 func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
 	rand.Seed(time.Now().UTC().UnixNano())
-	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second)
+	globalDNSCache = xhttp.NewDNSCache(10*time.Second, 10*time.Second, logger.LogOnceIf)
+
+	initGlobalContext()
+
+	globalReplicationState = newReplicationState()
+	globalTransitionState = newTransitionState()
+
+	console.SetColor("Debug", color.New())
 
 	gob.Register(StorageErr(""))
 }
@@ -64,10 +76,12 @@ func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
 		}
 	}
 
+	globalCompressConfigMu.Lock()
 	if globalCompressConfig.Enabled && !objAPI.IsCompressionSupported() {
 		logger.Fatal(errInvalidArgument,
 			"Compression support is requested but '%s' does not support compression", name)
 	}
+	globalCompressConfigMu.Unlock()
 }
 
 // Check for updates and print a notification message
@@ -272,6 +286,16 @@ func handleCommonEnvVars() {
 		globalConfigEncrypted = true
 	}
 
+	if env.IsSet(config.EnvRootUser) || env.IsSet(config.EnvRootPassword) {
+		cred, err := auth.CreateCredentials(env.Get(config.EnvRootUser, ""), env.Get(config.EnvRootPassword, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the shell environment")
+		}
+		globalActiveCred = cred
+		globalConfigEncrypted = true
+	}
+
 	if env.IsSet(config.EnvAccessKeyOld) && env.IsSet(config.EnvSecretKeyOld) {
 		oldCred, err := auth.CreateCredentials(env.Get(config.EnvAccessKeyOld, ""), env.Get(config.EnvSecretKeyOld, ""))
 		if err != nil {
@@ -282,6 +306,17 @@ func handleCommonEnvVars() {
 		os.Unsetenv(config.EnvAccessKeyOld)
 		os.Unsetenv(config.EnvSecretKeyOld)
 	}
+
+	if env.IsSet(config.EnvRootUserOld) && env.IsSet(config.EnvRootPasswordOld) {
+		oldCred, err := auth.CreateCredentials(env.Get(config.EnvRootUserOld, ""), env.Get(config.EnvRootPasswordOld, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate the old credentials inherited from the shell environment")
+		}
+		globalOldCred = oldCred
+		os.Unsetenv(config.EnvRootUserOld)
+		os.Unsetenv(config.EnvRootPasswordOld)
+	}
 }
 
 func logStartupMessage(msg string) {

cmd/config-current.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"sync"
@@ -25,6 +26,7 @@ import (
 	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
+	"github.com/minio/minio/cmd/config/crawler"
 	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/config/heal"
@@ -57,6 +59,7 @@ func initHelp() {
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
 		config.HealSubSys:           heal.DefaultKVS,
+		config.CrawlerSubSys:        crawler.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -112,6 +115,10 @@ func initHelp() {
 			Key:         config.HealSubSys,
 			Description: "manage object healing frequency and bitrot verification checks",
 		},
+		config.HelpKV{
+			Key:         config.CrawlerSubSys,
+			Description: "manage crawling for usage calculation, lifecycle, healing and more",
+		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
 			Description:     "send server logs to webhook endpoints",
@@ -192,6 +199,7 @@ func initHelp() {
 		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
 		config.HealSubSys:           heal.Help,
+		config.CrawlerSubSys:        crawler.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.PolicyOPASubSys:      opa.Help,
@@ -221,6 +229,9 @@ var (
 )
 
 func validateConfig(s config.Config, setDriveCount int) error {
+	// We must have a global lock for this so nobody else modifies env while we do.
+	defer env.LockSetEnv()()
+
 	// Disable merging env values with config for validation.
 	env.SetEnvOff()
 
@@ -249,14 +260,25 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		return err
 	}
 
-	if _, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]); err != nil {
+	compCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
+	if err != nil {
 		return err
 	}
+	objAPI := newObjectLayerFn()
+	if objAPI != nil {
+		if compCfg.Enabled && !objAPI.IsCompressionSupported() {
+			return fmt.Errorf("Backend does not support compression")
+		}
+	}
 
 	if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
 		return err
 	}
 
+	if _, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default]); err != nil {
+		return err
+	}
+
 	{
 		etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 		if err != nil {
@@ -438,10 +460,6 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 			}
 		}
 	}
-	globalHealConfig, err = heal.LookupConfig(s[config.HealSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to read heal config: %w", err))
-	}
 
 	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
 	if err != nil {
@@ -459,11 +477,6 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		logger.LogIf(ctx, fmt.Errorf("%s env is deprecated please migrate to using `mc encrypt` at bucket level", crypto.EnvKMSAutoEncryption))
 	}
 
-	globalCompressConfig, err = compress.LookupConfig(s[config.CompressionSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to setup Compression: %w", err))
-	}
-
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
 		NewGatewayHTTPTransport(), xhttp.DrainBody)
 	if err != nil {
@@ -538,6 +551,68 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
+
+	// Apply dynamic config values
+	logger.LogIf(ctx, applyDynamicConfig(ctx, s))
+}
+
+// applyDynamicConfig will apply dynamic config values.
+// Dynamic systems should be in config.SubSystemsDynamic as well.
+func applyDynamicConfig(ctx context.Context, s config.Config) error {
+	// Read all dynamic configs.
+	// API
+	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
+	}
+
+	// Compression
+	cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
+	if err != nil {
+		return fmt.Errorf("Unable to setup Compression: %w", err)
+	}
+	objAPI := newObjectLayerFn()
+	if objAPI != nil {
+		if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
+			return fmt.Errorf("Backend does not support compression")
+		}
+	}
+
+	// Heal
+	healCfg, err := heal.LookupConfig(s[config.HealSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to apply heal config: %w", err))
+	}
+
+	// Crawler
+	crawlerCfg, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default])
+	if err != nil {
+		return fmt.Errorf("Unable to apply crawler config: %w", err)
+	}
+
+	// Apply configurations.
+	// We should not fail after this.
+	globalAPIConfig.init(apiConfig, globalAPIConfig.setDriveCount)
+
+	globalCompressConfigMu.Lock()
+	globalCompressConfig = cmpCfg
+	globalCompressConfigMu.Unlock()
+
+	globalHealConfigMu.Lock()
+	globalHealConfig = healCfg
+	globalHealConfigMu.Unlock()
+
+	logger.LogIf(ctx, crawlerSleeper.Update(crawlerCfg.Delay, crawlerCfg.MaxWait))
+
+	// Update all dynamic config values in memory.
+	globalServerConfigMu.Lock()
+	defer globalServerConfigMu.Unlock()
+	if globalServerConfig != nil {
+		for k := range config.SubSystemsDynamic {
+			globalServerConfig[k] = s[k]
+		}
+	}
+	return nil
 }
 
 // Help - return sub-system level help

cmd/config-dir.go

@@ -87,7 +87,7 @@ func mkdirAllIgnorePerm(path string) error {
 	if err != nil {
 		// It is possible in kubernetes like deployments this directory
 		// is already mounted and is not writable, ignore any write errors.
-		if os.IsPermission(err) {
+		if osIsPermission(err) {
 			err = nil
 		}
 	}

cmd/config-encrypted.go

@@ -27,7 +27,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/madmin"
-	etcd "go.etcd.io/etcd/v3/clientv3"
+	etcd "go.etcd.io/etcd/clientv3"
 )
 
 func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
@@ -207,7 +207,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 	}
 
 	if encrypted && globalActiveCred.IsValid() && globalOldCred.IsValid() {
-		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
+		logger.Info("Rotation complete, please make sure to unset MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD envs")
 	}
 
 	return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationComplete)
@@ -294,7 +294,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 	}
 
 	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
-		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
+		logger.Info("Rotation complete, please make sure to unset MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD envs")
 	}
 
 	return saveConfig(GlobalContext, objAPI, backendEncryptedFile, backendEncryptedMigrationComplete)

cmd/config-migrate.go

@@ -74,7 +74,7 @@ func migrateConfig() error {
 	// Load only config version information.
 	version, err := GetVersion(getConfigFile())
 	if err != nil {
-		if os.IsNotExist(err) || os.IsPermission(err) {
+		if osIsNotExist(err) || osIsPermission(err) {
 			return nil
 		}
 		return err
@@ -243,7 +243,7 @@ func purgeV1() error {
 
 	cv1 := &configV1{}
 	_, err := Load(configFile, cv1)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %w", err)
@@ -264,7 +264,7 @@ func migrateV2ToV3() error {
 
 	cv2 := &configV2{}
 	_, err := Load(configFile, cv2)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘2’. %w", err)
@@ -323,7 +323,7 @@ func migrateV3ToV4() error {
 
 	cv3 := &configV3{}
 	_, err := Load(configFile, cv3)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘3’. %w", err)
@@ -361,7 +361,7 @@ func migrateV4ToV5() error {
 
 	cv4 := &configV4{}
 	_, err := Load(configFile, cv4)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘4’. %w", err)
@@ -402,7 +402,7 @@ func migrateV5ToV6() error {
 
 	cv5 := &configV5{}
 	_, err := Load(configFile, cv5)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘5’. %w", err)
@@ -491,7 +491,7 @@ func migrateV6ToV7() error {
 
 	cv6 := &configV6{}
 	_, err := Load(configFile, cv6)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘6’. %w", err)
@@ -547,7 +547,7 @@ func migrateV7ToV8() error {
 
 	cv7 := &serverConfigV7{}
 	_, err := Load(configFile, cv7)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘7’. %w", err)
@@ -609,7 +609,7 @@ func migrateV8ToV9() error {
 
 	cv8 := &serverConfigV8{}
 	_, err := Load(configFile, cv8)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘8’. %w", err)
@@ -679,7 +679,7 @@ func migrateV9ToV10() error {
 
 	cv9 := &serverConfigV9{}
 	_, err := Load(configFile, cv9)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘9’. %w", err)
@@ -747,7 +747,7 @@ func migrateV10ToV11() error {
 
 	cv10 := &serverConfigV10{}
 	_, err := Load(configFile, cv10)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘10’. %w", err)
@@ -818,7 +818,7 @@ func migrateV11ToV12() error {
 
 	cv11 := &serverConfigV11{}
 	_, err := Load(configFile, cv11)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘11’. %w", err)
@@ -915,7 +915,7 @@ func migrateV12ToV13() error {
 
 	cv12 := &serverConfigV12{}
 	_, err := Load(configFile, cv12)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘12’. %w", err)
@@ -995,7 +995,7 @@ func migrateV13ToV14() error {
 
 	cv13 := &serverConfigV13{}
 	_, err := Load(configFile, cv13)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘13’. %w", err)
@@ -1080,7 +1080,7 @@ func migrateV14ToV15() error {
 
 	cv14 := &serverConfigV14{}
 	_, err := Load(configFile, cv14)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘14’. %w", err)
@@ -1170,7 +1170,7 @@ func migrateV15ToV16() error {
 
 	cv15 := &serverConfigV15{}
 	_, err := Load(configFile, cv15)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘15’. %w", err)
@@ -1260,7 +1260,7 @@ func migrateV16ToV17() error {
 
 	cv16 := &serverConfigV16{}
 	_, err := Load(configFile, cv16)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘16’. %w", err)
@@ -1381,7 +1381,7 @@ func migrateV17ToV18() error {
 
 	cv17 := &serverConfigV17{}
 	_, err := Load(configFile, cv17)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘17’. %w", err)
@@ -1483,7 +1483,7 @@ func migrateV18ToV19() error {
 
 	cv18 := &serverConfigV18{}
 	_, err := Load(configFile, cv18)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘18’. %w", err)
@@ -1589,7 +1589,7 @@ func migrateV19ToV20() error {
 
 	cv19 := &serverConfigV19{}
 	_, err := Load(configFile, cv19)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘18’. %w", err)
@@ -1694,7 +1694,7 @@ func migrateV20ToV21() error {
 
 	cv20 := &serverConfigV20{}
 	_, err := Load(configFile, cv20)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘20’. %w", err)
@@ -1798,7 +1798,7 @@ func migrateV21ToV22() error {
 
 	cv21 := &serverConfigV21{}
 	_, err := Load(configFile, cv21)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘21’. %w", err)
@@ -1902,7 +1902,7 @@ func migrateV22ToV23() error {
 
 	cv22 := &serverConfigV22{}
 	_, err := Load(configFile, cv22)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘22’. %w", err)
@@ -2015,7 +2015,7 @@ func migrateV23ToV24() error {
 
 	cv23 := &serverConfigV23{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv23)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘23’. %w", err)
@@ -2128,7 +2128,7 @@ func migrateV24ToV25() error {
 
 	cv24 := &serverConfigV24{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv24)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘24’. %w", err)
@@ -2246,7 +2246,7 @@ func migrateV25ToV26() error {
 
 	cv25 := &serverConfigV25{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘25’. %w", err)
@@ -2368,7 +2368,7 @@ func migrateV26ToV27() error {
 	// in the new `logger` field
 	srvConfig := &serverConfigV27{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %w", err)
@@ -2401,7 +2401,7 @@ func migrateV27ToV28() error {
 
 	srvConfig := &serverConfigV28{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %w", err)
@@ -2452,17 +2452,17 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	var config = &serverConfigV27{}
 	for _, cfgFile := range configFiles {
 		if _, err = Load(cfgFile, config); err != nil {
-			if !os.IsNotExist(err) && !os.IsPermission(err) {
+			if !osIsNotExist(err) && !osIsPermission(err) {
 				return err
 			}
 			continue
 		}
 		break
 	}
-	if os.IsPermission(err) {
+	if osIsPermission(err) {
 		logger.Info("Older config found but not readable %s, proceeding to initialize new config anyways", err)
 	}
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		// Initialize the server config, if no config exists.
 		return newSrvConfig(objAPI)
 	}

cmd/config-migrate_test.go

@@ -60,7 +60,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	}
 
 	// Check if config v1 is removed from filesystem
-	if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
+	if _, err := os.Stat(configPath); err == nil || !osIsNotExist(err) {
 		t.Fatal("Config V1 file is not purged")
 	}
 

cmd/config/api/api.go

@@ -34,12 +34,17 @@ const (
 	apiClusterDeadline         = "cluster_deadline"
 	apiCorsAllowOrigin         = "cors_allow_origin"
 	apiRemoteTransportDeadline = "remote_transport_deadline"
+	apiListQuorum              = "list_quorum"
+	apiExtendListCacheLife     = "extend_list_cache_life"
 
 	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
 	EnvAPIClusterDeadline         = "MINIO_API_CLUSTER_DEADLINE"
 	EnvAPICorsAllowOrigin         = "MINIO_API_CORS_ALLOW_ORIGIN"
 	EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
+	EnvAPIListQuorum              = "MINIO_API_LIST_QUORUM"
+	EnvAPIExtendListCacheLife     = "MINIO_API_EXTEND_LIST_CACHE_LIFE"
+	EnvAPISecureCiphers           = "MINIO_API_SECURE_CIPHERS"
 )
 
 // Deprecated key and ENVs
@@ -71,6 +76,14 @@ var (
 			Key:   apiRemoteTransportDeadline,
 			Value: "2h",
 		},
+		config.KV{
+			Key:   apiListQuorum,
+			Value: "optimal",
+		},
+		config.KV{
+			Key:   apiExtendListCacheLife,
+			Value: "0s",
+		},
 	}
 )
 
@@ -81,6 +94,8 @@ type Config struct {
 	ClusterDeadline         time.Duration `json:"cluster_deadline"`
 	CorsAllowOrigin         []string      `json:"cors_allow_origin"`
 	RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
+	ListQuorum              string        `json:"list_strict_quorum"`
+	ExtendListLife          time.Duration `json:"extend_list_cache_life"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -94,6 +109,22 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 	return json.Unmarshal(data, &aux)
 }
 
+// GetListQuorum interprets list quorum values and returns appropriate
+// acceptable quorum expected for list operations
+func (sCfg Config) GetListQuorum() int {
+	switch sCfg.ListQuorum {
+	case "reduced":
+		return 2
+	case "disk":
+		// smallest possible value, generally meant for testing.
+		return 1
+	case "strict":
+		return -1
+	}
+	// Defaults to 3 drives per set, defaults to "optimal" value
+	return 3
+}
+
 // LookupConfig - lookup api config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	// remove this since we have removed this already.
@@ -130,11 +161,25 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
+	listQuorum := env.Get(EnvAPIListQuorum, kvs.Get(apiListQuorum))
+	switch listQuorum {
+	case "strict", "optimal", "reduced", "disk":
+	default:
+		return cfg, errors.New("invalid value for list strict quorum")
+	}
+
+	listLife, err := time.ParseDuration(env.Get(EnvAPIExtendListCacheLife, kvs.Get(apiExtendListCacheLife)))
+	if err != nil {
+		return cfg, err
+	}
+
 	return Config{
 		RequestsMax:             requestsMax,
 		RequestsDeadline:        requestsDeadline,
 		ClusterDeadline:         clusterDeadline,
 		CorsAllowOrigin:         corsAllowOrigin,
 		RemoteTransportDeadline: remoteTransportDeadline,
+		ListQuorum:              listQuorum,
+		ExtendListLife:          listLife,
 	}, nil
 }

cmd/config/cache/config.go

@@ -28,16 +28,17 @@ import (
 
 // Config represents cache config settings
 type Config struct {
-	Enabled       bool     `json:"-"`
-	Drives        []string `json:"drives"`
-	Expiry        int      `json:"expiry"`
-	MaxUse        int      `json:"maxuse"`
-	Quota         int      `json:"quota"`
-	Exclude       []string `json:"exclude"`
-	After         int      `json:"after"`
-	WatermarkLow  int      `json:"watermark_low"`
-	WatermarkHigh int      `json:"watermark_high"`
-	Range         bool     `json:"range"`
+	Enabled         bool     `json:"-"`
+	Drives          []string `json:"drives"`
+	Expiry          int      `json:"expiry"`
+	MaxUse          int      `json:"maxuse"`
+	Quota           int      `json:"quota"`
+	Exclude         []string `json:"exclude"`
+	After           int      `json:"after"`
+	WatermarkLow    int      `json:"watermark_low"`
+	WatermarkHigh   int      `json:"watermark_high"`
+	Range           bool     `json:"range"`
+	CommitWriteback bool     `json:"-"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
@@ -152,3 +153,13 @@ func parseCacheExcludes(excludes string) ([]string, error) {
 
 	return excludesSlice, nil
 }
+
+func parseCacheCommitMode(commitStr string) (bool, error) {
+	switch strings.ToLower(commitStr) {
+	case "writeback":
+		return true, nil
+	case "writethrough":
+		return false, nil
+	}
+	return false, config.ErrInvalidCacheCommitValue(nil).Msg("cache commit value must be `writeback` or `writethrough`")
+}

cmd/config/cache/help.go

@@ -74,5 +74,11 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         Commit,
+			Description: `set to control cache commit behavior, defaults to "writethrough"`,
+			Optional:    true,
+			Type:        "string",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -35,6 +35,7 @@ const (
 	WatermarkLow  = "watermark_low"
 	WatermarkHigh = "watermark_high"
 	Range         = "range"
+	Commit        = "commit"
 
 	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
 	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
@@ -45,6 +46,7 @@ const (
 	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
 	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
 	EnvCacheRange         = "MINIO_CACHE_RANGE"
+	EnvCacheCommit        = "MINIO_CACHE_COMMIT"
 
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
@@ -53,6 +55,7 @@ const (
 	DefaultAfter         = "0"
 	DefaultWaterMarkLow  = "70"
 	DefaultWaterMarkHigh = "80"
+	DefaultCacheCommit   = "writethrough"
 )
 
 // DefaultKVS - default KV settings for caching.
@@ -90,6 +93,10 @@ var (
 			Key:   Range,
 			Value: config.EnableOn,
 		},
+		config.KV{
+			Key:   Commit,
+			Value: DefaultCacheCommit,
+		},
 	}
 )
 
@@ -210,6 +217,16 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		}
 		cfg.Range = rng
 	}
+	if commit := env.Get(EnvCacheCommit, kvs.Get(Commit)); commit != "" {
+		cfg.CommitWriteback, err = parseCacheCommitMode(commit)
+		if err != nil {
+			return cfg, err
+		}
+		if cfg.After > 0 && cfg.CommitWriteback {
+			err := errors.New("cache after cannot be used with commit writeback")
+			return cfg, config.ErrInvalidCacheSetting(err)
+		}
+	}
 
 	return cfg, nil
 }

cmd/config/compress/compress.go

@@ -26,19 +26,22 @@ import (
 
 // Config represents the compression settings.
 type Config struct {
-	Enabled    bool     `json:"enabled"`
-	Extensions []string `json:"extensions"`
-	MimeTypes  []string `json:"mime-types"`
+	Enabled        bool     `json:"enabled"`
+	AllowEncrypted bool     `json:"allow_encryption"`
+	Extensions     []string `json:"extensions"`
+	MimeTypes      []string `json:"mime-types"`
 }
 
 // Compression environment variables
 const (
-	Extensions = "extensions"
-	MimeTypes  = "mime_types"
+	Extensions     = "extensions"
+	AllowEncrypted = "allow_encryption"
+	MimeTypes      = "mime_types"
 
-	EnvCompressState      = "MINIO_COMPRESS_ENABLE"
-	EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
-	EnvCompressMimeTypes  = "MINIO_COMPRESS_MIME_TYPES"
+	EnvCompressState           = "MINIO_COMPRESS_ENABLE"
+	EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
+	EnvCompressExtensions      = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressMimeTypes       = "MINIO_COMPRESS_MIME_TYPES"
 
 	// Include-list for compression.
 	DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
@@ -52,6 +55,10 @@ var (
 			Key:   config.Enable,
 			Value: config.EnableOff,
 		},
+		config.KV{
+			Key:   AllowEncrypted,
+			Value: config.EnableOff,
+		},
 		config.KV{
 			Key:   Extensions,
 			Value: DefaultExtensions,
@@ -101,6 +108,12 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		return cfg, nil
 	}
 
+	allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted))
+	cfg.AllowEncrypted, err = config.ParseBool(allowEnc)
+	if err != nil {
+		return cfg, err
+	}
+
 	compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
 	compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
 	compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))

cmd/config/config.go

@@ -77,6 +77,7 @@ const (
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
 	HealSubSys           = "heal"
+	CrawlerSubSys        = "crawler"
 
 	// Add new constants here if you add new fields to config.
 )
@@ -98,7 +99,7 @@ const (
 )
 
 // SubSystems - all supported sub-systems
-var SubSystems = set.CreateStringSet([]string{
+var SubSystems = set.CreateStringSet(
 	CredentialsSubSys,
 	RegionSubSys,
 	EtcdSubSys,
@@ -113,6 +114,7 @@ var SubSystems = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
+	CrawlerSubSys,
 	HealSubSys,
 	NotifyAMQPSubSys,
 	NotifyESSubSys,
@@ -124,7 +126,15 @@ var SubSystems = set.CreateStringSet([]string{
 	NotifyPostgresSubSys,
 	NotifyRedisSubSys,
 	NotifyWebhookSubSys,
-}...)
+)
+
+// SubSystemsDynamic - all sub-systems that have dynamic config.
+var SubSystemsDynamic = set.CreateStringSet(
+	APISubSys,
+	CompressionSubSys,
+	CrawlerSubSys,
+	HealSubSys,
+)
 
 // SubSystemsSingleTargets - subsystems which only support single target.
 var SubSystemsSingleTargets = set.CreateStringSet([]string{
@@ -141,6 +151,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
 	HealSubSys,
+	CrawlerSubSys,
 }...)
 
 // Constant separators
@@ -309,25 +320,29 @@ func (c Config) DelFrom(r io.Reader) error {
 	return nil
 }
 
-// ReadFrom - implements io.ReaderFrom interface
-func (c Config) ReadFrom(r io.Reader) (int64, error) {
+// ReadConfig - read content from input and write into c.
+// Returns whether all parameters were dynamic.
+func (c Config) ReadConfig(r io.Reader) (dynOnly bool, err error) {
 	var n int
 	scanner := bufio.NewScanner(r)
+	dynOnly = true
 	for scanner.Scan() {
 		// Skip any empty lines, or comment like characters
 		text := scanner.Text()
 		if text == "" || strings.HasPrefix(text, KvComment) {
 			continue
 		}
-		if err := c.SetKVS(text, DefaultKVS); err != nil {
-			return 0, err
+		dynamic, err := c.SetKVS(text, DefaultKVS)
+		if err != nil {
+			return false, err
 		}
+		dynOnly = dynOnly && dynamic
 		n += len(text)
 	}
 	if err := scanner.Err(); err != nil {
-		return 0, err
+		return false, err
 	}
-	return int64(n), nil
+	return dynOnly, nil
 }
 
 type configWriteTo struct {
@@ -618,26 +633,27 @@ func (c Config) Clone() Config {
 }
 
 // SetKVS - set specific key values per sub-system.
-func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
+func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err error) {
 	if len(s) == 0 {
-		return Errorf("input arguments cannot be empty")
+		return false, Errorf("input arguments cannot be empty")
 	}
 	inputs := strings.SplitN(s, KvSpaceSeparator, 2)
 	if len(inputs) <= 1 {
-		return Errorf("invalid number of arguments '%s'", s)
+		return false, Errorf("invalid number of arguments '%s'", s)
 	}
 	subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
 	if len(subSystemValue) == 0 {
-		return Errorf("invalid number of arguments %s", s)
+		return false, Errorf("invalid number of arguments %s", s)
 	}
 
 	if !SubSystems.Contains(subSystemValue[0]) {
-		return Errorf("unknown sub-system %s", s)
+		return false, Errorf("unknown sub-system %s", s)
 	}
 
 	if SubSystemsSingleTargets.Contains(subSystemValue[0]) && len(subSystemValue) == 2 {
-		return Errorf("sub-system '%s' only supports single target", subSystemValue[0])
+		return false, Errorf("sub-system '%s' only supports single target", subSystemValue[0])
 	}
+	dynamic = SubSystemsDynamic.Contains(subSystemValue[0])
 
 	tgt := Default
 	subSys := subSystemValue[0]
@@ -647,7 +663,7 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 
 	fields := madmin.KvFields(inputs[1], defaultKVS[subSys].Keys())
 	if len(fields) == 0 {
-		return Errorf("sub-system '%s' cannot have empty keys", subSys)
+		return false, Errorf("sub-system '%s' cannot have empty keys", subSys)
 	}
 
 	var kvs = KVS{}
@@ -670,7 +686,7 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 			kvs.Set(prevK, madmin.SanitizeValue(kv[1]))
 			continue
 		}
-		return Errorf("key '%s', cannot have empty value", kv[0])
+		return false, Errorf("key '%s', cannot have empty value", kv[0])
 	}
 
 	_, ok := kvs.Lookup(Enable)
@@ -720,11 +736,11 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 			// Return error only if the
 			// key is enabled, for state=off
 			// let it be empty.
-			return Errorf(
+			return false, Errorf(
 				"'%s' is not optional for '%s' sub-system, please check '%s' documentation",
 				hkv.Key, subSys, subSys)
 		}
 	}
 	c[subSys][tgt] = currKVS
-	return nil
+	return dynamic, nil
 }

cmd/config/constants.go

@@ -23,17 +23,21 @@ const (
 
 // Top level common ENVs
 const (
-	EnvAccessKey    = "MINIO_ACCESS_KEY"
-	EnvSecretKey    = "MINIO_SECRET_KEY"
-	EnvAccessKeyOld = "MINIO_ACCESS_KEY_OLD"
-	EnvSecretKeyOld = "MINIO_SECRET_KEY_OLD"
-	EnvBrowser      = "MINIO_BROWSER"
-	EnvDomain       = "MINIO_DOMAIN"
-	EnvRegionName   = "MINIO_REGION_NAME"
-	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
-	EnvFSOSync      = "MINIO_FS_OSYNC"
-	EnvArgs         = "MINIO_ARGS"
-	EnvDNSWebhook   = "MINIO_DNS_WEBHOOK_ENDPOINT"
+	EnvAccessKey       = "MINIO_ACCESS_KEY"
+	EnvSecretKey       = "MINIO_SECRET_KEY"
+	EnvRootUser        = "MINIO_ROOT_USER"
+	EnvRootPassword    = "MINIO_ROOT_PASSWORD"
+	EnvAccessKeyOld    = "MINIO_ACCESS_KEY_OLD"
+	EnvSecretKeyOld    = "MINIO_SECRET_KEY_OLD"
+	EnvRootUserOld     = "MINIO_ROOT_USER_OLD"
+	EnvRootPasswordOld = "MINIO_ROOT_PASSWORD_OLD"
+	EnvBrowser         = "MINIO_BROWSER"
+	EnvDomain          = "MINIO_DOMAIN"
+	EnvRegionName      = "MINIO_REGION_NAME"
+	EnvPublicIPs       = "MINIO_PUBLIC_IPS"
+	EnvFSOSync         = "MINIO_FS_OSYNC"
+	EnvArgs            = "MINIO_ARGS"
+	EnvDNSWebhook      = "MINIO_DNS_WEBHOOK_ENDPOINT"
 
 	EnvUpdate = "MINIO_UPDATE"
 

cmd/config/crawler/crawler.go

@@ -0,0 +1,88 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawler
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Compression environment variables
+const (
+	Delay   = "delay"
+	MaxWait = "max_wait"
+
+	EnvDelay   = "MINIO_CRAWLER_DELAY"
+	EnvMaxWait = "MINIO_CRAWLER_MAX_WAIT"
+)
+
+// Config represents the heal settings.
+type Config struct {
+	// Delay is the sleep multiplier.
+	Delay float64 `json:"delay"`
+	// MaxWait is maximum wait time between operations
+	MaxWait time.Duration
+}
+
+var (
+	// DefaultKVS - default KV config for heal settings
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   Delay,
+			Value: "10",
+		},
+		config.KV{
+			Key:   MaxWait,
+			Value: "15s",
+		},
+	}
+
+	// Help provides help for config values
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         Delay,
+			Description: `crawler delay multiplier, defaults to '10.0'`,
+			Optional:    true,
+			Type:        "float",
+		},
+		config.HelpKV{
+			Key:         MaxWait,
+			Description: `maximum wait time between operations, defaults to '15s'`,
+			Optional:    true,
+			Type:        "duration",
+		},
+	}
+)
+
+// LookupConfig - lookup config and override with valid environment settings if any.
+func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	if err = config.CheckValidKeys(config.CrawlerSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+	cfg.Delay, err = strconv.ParseFloat(env.Get(EnvDelay, kvs.Get(Delay)), 64)
+	if err != nil {
+		return cfg, err
+	}
+	cfg.MaxWait, err = time.ParseDuration(env.Get(EnvMaxWait, kvs.Get(MaxWait)))
+	if err != nil {
+		return cfg, err
+	}
+	return cfg, nil
+}

cmd/config/dns/etcd_dns.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/coredns/coredns/plugin/etcd/msg"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"go.etcd.io/etcd/v3/clientv3"
+	"go.etcd.io/etcd/clientv3"
 )
 
 // ErrNoEntriesFound - Indicates no entries were found for the given key (directory)

cmd/config/errors.go

@@ -102,22 +102,33 @@ var (
 		"MINIO_CACHE_RANGE: Valid expected value is `on` or `off`",
 	)
 
+	ErrInvalidCacheCommitValue = newErrFn(
+		"Invalid cache commit value",
+		"Please check the passed value",
+		"MINIO_CACHE_COMMIT: Valid expected value is `writeback` or `writethrough`",
+	)
+
+	ErrInvalidCacheSetting = newErrFn(
+		"Incompatible cache setting",
+		"Please check the passed value",
+		"MINIO_CACHE_AFTER cannot be used with MINIO_CACHE_COMMIT setting",
+	)
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",
-		`Detected encrypted config backend, correct old access and secret keys should be specified via environment variables MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD to be able to re-encrypt the MinIO config, user IAM and policies with new credentials`,
+		`Detected encrypted config backend, correct old access and secret keys should be specified via environment variables MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD to be able to re-encrypt the MinIO config, user IAM and policies with new credentials`,
 	)
 
 	ErrInvalidCredentialsBackendEncrypted = newErrFn(
 		"Invalid credentials",
 		"Please set correct credentials in the environment for decryption",
-		`Detected encrypted config backend, correct access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be able to decrypt the MinIO config, user IAM and policies`,
+		`Detected encrypted config backend, correct access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD to be able to decrypt the MinIO config, user IAM and policies`,
 	)
 
 	ErrMissingCredentialsBackendEncrypted = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
-		`Detected encrypted config backend, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be able to decrypt the MinIO config, user IAM and policies`,
+		`Detected encrypted config backend, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD to be able to decrypt the MinIO config, user IAM and policies`,
 	)
 
 	ErrInvalidCredentials = newErrFn(
@@ -129,13 +140,13 @@ var (
 	ErrEnvCredentialsMissingGateway = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
-		`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
+		`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
 	)
 
 	ErrEnvCredentialsMissingDistributed = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
-		`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
+		`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
 	)
 
 	ErrInvalidErasureEndpoints = newErrFn(

cmd/config/etcd/etcd.go

@@ -25,8 +25,8 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
 	xnet "github.com/minio/minio/pkg/net"
-	"go.etcd.io/etcd/v3/clientv3"
-	"go.etcd.io/etcd/v3/clientv3/namespace"
+	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/clientv3/namespace"
 )
 
 const (

cmd/config/heal/heal.go

@@ -17,20 +17,32 @@
 package heal
 
 import (
-	"errors"
+	"fmt"
+	"strconv"
+	"time"
 
 	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
 )
 
 // Compression environment variables
 const (
-	Bitrot = "bitrotscan"
+	Bitrot  = "bitrotscan"
+	Sleep   = "max_sleep"
+	IOCount = "max_io"
+
+	EnvBitrot  = "MINIO_HEAL_BITROTSCAN"
+	EnvSleep   = "MINIO_HEAL_MAX_SLEEP"
+	EnvIOCount = "MINIO_HEAL_MAX_IO"
 )
 
 // Config represents the heal settings.
 type Config struct {
 	// Bitrot will perform bitrot scan on local disk when checking objects.
 	Bitrot bool `json:"bitrotscan"`
+	// maximum sleep duration between objects to slow down heal operation.
+	Sleep   time.Duration `json:"sleep"`
+	IOCount int           `json:"iocount"`
 }
 
 var (
@@ -40,6 +52,14 @@ var (
 			Key:   Bitrot,
 			Value: config.EnableOff,
 		},
+		config.KV{
+			Key:   Sleep,
+			Value: "1s",
+		},
+		config.KV{
+			Key:   IOCount,
+			Value: "10",
+		},
 	}
 
 	// Help provides help for config values
@@ -50,6 +70,18 @@ var (
 			Optional:    true,
 			Type:        "on|off",
 		},
+		config.HelpKV{
+			Key:         Sleep,
+			Description: `maximum sleep duration between objects to slow down heal operation. eg. 2s`,
+			Optional:    true,
+			Type:        "duration",
+		},
+		config.HelpKV{
+			Key:         IOCount,
+			Description: `maximum IO requests allowed between objects to slow down heal operation. eg. 3`,
+			Optional:    true,
+			Type:        "int",
+		},
 	}
 )
 
@@ -58,10 +90,17 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	if err = config.CheckValidKeys(config.HealSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
-	bitrot := kvs.Get(Bitrot)
-	if bitrot != config.EnableOn && bitrot != config.EnableOff {
-		return cfg, errors.New(Bitrot + ": must be 'on' or 'off'")
+	cfg.Bitrot, err = config.ParseBool(env.Get(EnvBitrot, kvs.Get(Bitrot)))
+	if err != nil {
+		return cfg, fmt.Errorf("'heal:bitrotscan' value invalid: %w", err)
+	}
+	cfg.Sleep, err = time.ParseDuration(env.Get(EnvSleep, kvs.Get(Sleep)))
+	if err != nil {
+		return cfg, fmt.Errorf("'heal:max_sleep' value invalid: %w", err)
+	}
+	cfg.IOCount, err = strconv.Atoi(env.Get(EnvIOCount, kvs.Get(IOCount)))
+	if err != nil {
+		return cfg, fmt.Errorf("'heal:max_io' value invalid: %w", err)
 	}
-	cfg.Bitrot = bitrot == config.EnableOn
 	return cfg, nil
 }

cmd/config/storageclass/help.go

@@ -33,6 +33,12 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         ClassDMA,
+			Description: `enable O_DIRECT for both read and write, defaults to "write" e.g. "read+write"`,
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

cmd/config/storageclass/storage-class.go

@@ -18,6 +18,7 @@ package storageclass
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -32,17 +33,26 @@ const (
 	RRS = "REDUCED_REDUNDANCY"
 	// Standard storage class
 	STANDARD = "STANDARD"
+	// DMA storage class
+	DMA = "DMA"
+
+	// Valid values are "write" and "read+write"
+	DMAWrite     = "write"
+	DMAReadWrite = "read+write"
 )
 
 // Standard constats for config info storage class
 const (
 	ClassStandard = "standard"
 	ClassRRS      = "rrs"
+	ClassDMA      = "dma"
 
 	// Reduced redundancy storage class environment variable
 	RRSEnv = "MINIO_STORAGE_CLASS_RRS"
 	// Standard storage class environment variable
 	StandardEnv = "MINIO_STORAGE_CLASS_STANDARD"
+	// DMA storage class environment variable
+	DMAEnv = "MINIO_STORAGE_CLASS_DMA"
 
 	// Supported storage class scheme is EC
 	schemePrefix = "EC"
@@ -52,6 +62,9 @@ const (
 
 	// Default RRS parity is always minimum parity.
 	defaultRRSParity = minParityDisks
+
+	// Default DMA value
+	defaultDMA = DMAWrite
 )
 
 // DefaultKVS - default storage class config
@@ -65,18 +78,24 @@ var (
 			Key:   ClassRRS,
 			Value: "EC:2",
 		},
+		config.KV{
+			Key:   ClassDMA,
+			Value: defaultDMA,
+		},
 	}
 )
 
 // StorageClass - holds storage class information
 type StorageClass struct {
 	Parity int
+	DMA    string
 }
 
 // Config storage class configuration
 type Config struct {
 	Standard StorageClass `json:"standard"`
 	RRS      StorageClass `json:"rrs"`
+	DMA      StorageClass `json:"dma"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -93,7 +112,7 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 // IsValid - returns true if input string is a valid
 // storage class kind supported.
 func IsValid(sc string) bool {
-	return sc == RRS || sc == STANDARD
+	return sc == RRS || sc == STANDARD || sc == DMA
 }
 
 // UnmarshalText unmarshals storage class from its textual form into
@@ -103,6 +122,14 @@ func (sc *StorageClass) UnmarshalText(b []byte) error {
 	if scStr == "" {
 		return nil
 	}
+	if scStr == DMAWrite {
+		sc.DMA = DMAWrite
+		return nil
+	}
+	if scStr == DMAReadWrite {
+		sc.DMA = DMAReadWrite
+		return nil
+	}
 	s, err := parseStorageClass(scStr)
 	if err != nil {
 		return err
@@ -116,14 +143,14 @@ func (sc *StorageClass) MarshalText() ([]byte, error) {
 	if sc.Parity != 0 {
 		return []byte(fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)), nil
 	}
-	return []byte(""), nil
+	return []byte(sc.DMA), nil
 }
 
 func (sc *StorageClass) String() string {
 	if sc.Parity != 0 {
 		return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
 	}
-	return ""
+	return sc.DMA
 }
 
 // Parses given storageClassEnv and returns a storageClass structure.
@@ -212,6 +239,11 @@ func (sCfg Config) GetParityForSC(sc string) (parity int) {
 	}
 }
 
+// GetDMA - returns DMA configuration.
+func (sCfg Config) GetDMA() string {
+	return sCfg.DMA.DMA
+}
+
 // Enabled returns if etcd is enabled.
 func Enabled(kvs config.KVS) bool {
 	ssc := kvs.Get(ClassStandard)
@@ -231,6 +263,7 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 
 	ssc := env.Get(StandardEnv, kvs.Get(ClassStandard))
 	rrsc := env.Get(RRSEnv, kvs.Get(ClassRRS))
+	dma := env.Get(DMAEnv, kvs.Get(ClassDMA))
 	// Check for environment variables and parse into storageClass struct
 	if ssc != "" {
 		cfg.Standard, err = parseStorageClass(ssc)
@@ -252,6 +285,14 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 		cfg.RRS.Parity = defaultRRSParity
 	}
 
+	if dma == "" {
+		dma = defaultDMA
+	}
+	if dma != DMAReadWrite && dma != DMAWrite {
+		return Config{}, errors.New(`valid dma values are "read-write" and "write"`)
+	}
+	cfg.DMA.DMA = dma
+
 	// Validation is done after parsing both the storage classes. This is needed because we need one
 	// storage class value to deduce the correct value of the other storage class.
 	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, setDriveCount); err != nil {

cmd/consolelogger.go

@@ -40,8 +40,8 @@ type HTTPConsoleLoggerSys struct {
 	logBuf   *ring.Ring
 }
 
-func mustGetNodeName(endpointServerSets EndpointServerSets) (nodeName string) {
-	host, err := xnet.ParseHost(GetLocalPeer(endpointServerSets))
+func mustGetNodeName(endpointServerPools EndpointServerPools) (nodeName string) {
+	host, err := xnet.ParseHost(GetLocalPeer(endpointServerPools))
 	if err != nil {
 		logger.FatalIf(err, "Unable to start console logging subsystem")
 	}
@@ -63,14 +63,14 @@ func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
 }
 
 // SetNodeName - sets the node name if any after distributed setup has initialized
-func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointServerSets EndpointServerSets) {
-	sys.nodeName = mustGetNodeName(endpointServerSets)
+func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointServerPools EndpointServerPools) {
+	sys.nodeName = mustGetNodeName(endpointServerPools)
 }
 
 // HasLogListeners returns true if console log listeners are registered
 // for this node or peers
 func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
-	return sys != nil && sys.pubsub.HasSubscribers()
+	return sys != nil && sys.pubsub.NumSubscribers() > 0
 }
 
 // Subscribe starts console logging for this node.

cmd/crypto/header.go

@@ -19,231 +19,62 @@ import (
 	"crypto/md5"
 	"encoding/base64"
 	"net/http"
-	"strings"
 
-	jsoniter "github.com/json-iterator/go"
 	xhttp "github.com/minio/minio/cmd/http"
 )
 
-// SSEHeader is the general AWS SSE HTTP header key.
-const SSEHeader = "X-Amz-Server-Side-Encryption"
-
-const (
-	// SSEKmsID is the HTTP header key referencing the SSE-KMS
-	// key ID.
-	SSEKmsID = SSEHeader + "-Aws-Kms-Key-Id"
-
-	// SSEKmsContext is the HTTP header key referencing the
-	// SSE-KMS encryption context.
-	SSEKmsContext = SSEHeader + "-Context"
-)
-
-const (
-	// SSECAlgorithm is the HTTP header key referencing
-	// the SSE-C algorithm.
-	SSECAlgorithm = SSEHeader + "-Customer-Algorithm"
-
-	// SSECKey is the HTTP header key referencing the
-	// SSE-C client-provided key..
-	SSECKey = SSEHeader + "-Customer-Key"
-
-	// SSECKeyMD5 is the HTTP header key referencing
-	// the MD5 sum of the client-provided key.
-	SSECKeyMD5 = SSEHeader + "-Customer-Key-Md5"
-)
-
-const (
-	// SSECopyAlgorithm is the HTTP header key referencing
-	// the SSE-C algorithm for SSE-C copy requests.
-	SSECopyAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
-
-	// SSECopyKey is the HTTP header key referencing the SSE-C
-	// client-provided key for SSE-C copy requests.
-	SSECopyKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
-
-	// SSECopyKeyMD5 is the HTTP header key referencing the
-	// MD5 sum of the client key for SSE-C copy requests.
-	SSECopyKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5"
-)
-
-const (
-	// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
-	// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
-	// For SSE-C  see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
-	SSEAlgorithmAES256 = "AES256"
-
-	// SSEAlgorithmKMS is the value of 'X-Amz-Server-Side-Encryption' for SSE-KMS.
-	// See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
-	SSEAlgorithmKMS = "aws:kms"
-)
-
 // RemoveSensitiveHeaders removes confidential encryption
 // information - e.g. the SSE-C key - from the HTTP headers.
 // It has the same semantics as RemoveSensitiveEntires.
 func RemoveSensitiveHeaders(h http.Header) {
-	h.Del(SSECKey)
-	h.Del(SSECopyKey)
+	h.Del(xhttp.AmzServerSideEncryptionCustomerKey)
+	h.Del(xhttp.AmzServerSideEncryptionCopyCustomerKey)
 	h.Del(xhttp.AmzMetaUnencryptedContentLength)
 	h.Del(xhttp.AmzMetaUnencryptedContentMD5)
 }
 
-// IsRequested returns true if the HTTP headers indicates
-// that any form server-side encryption (SSE-C, SSE-S3 or SSE-KMS)
-// is requested.
-func IsRequested(h http.Header) bool {
-	return S3.IsRequested(h) || SSEC.IsRequested(h) || SSECopy.IsRequested(h) || S3KMS.IsRequested(h)
-}
-
-// S3 represents AWS SSE-S3. It provides functionality to handle
-// SSE-S3 requests.
-var S3 = s3{}
-
-type s3 struct{}
-
-// IsRequested returns true if the HTTP headers indicates that
-// the S3 client requests SSE-S3.
-func (s3) IsRequested(h http.Header) bool {
-	_, ok := h[SSEHeader]
-	return ok && strings.ToLower(h.Get(SSEHeader)) != SSEAlgorithmKMS // Return only true if the SSE header is specified and does not contain the SSE-KMS value
-}
-
-// ParseHTTP parses the SSE-S3 related HTTP headers and checks
-// whether they contain valid values.
-func (s3) ParseHTTP(h http.Header) (err error) {
-	if h.Get(SSEHeader) != SSEAlgorithmAES256 {
-		err = ErrInvalidEncryptionMethod
-	}
-	return
-}
-
-// S3KMS represents AWS SSE-KMS. It provides functionality to
-// handle SSE-KMS requests.
-var S3KMS = s3KMS{}
-
-type s3KMS struct{}
-
-// IsRequested returns true if the HTTP headers indicates that
-// the S3 client requests SSE-KMS.
-func (s3KMS) IsRequested(h http.Header) bool {
-	if _, ok := h[SSEKmsID]; ok {
-		return true
-	}
-	if _, ok := h[SSEKmsContext]; ok {
-		return true
-	}
-	if _, ok := h[SSEHeader]; ok {
-		return strings.ToUpper(h.Get(SSEHeader)) != SSEAlgorithmAES256 // Return only true if the SSE header is specified and does not contain the SSE-S3 value
-	}
-	return false
-}
-
-// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
-// and context, if present, on success.
-func (s3KMS) ParseHTTP(h http.Header) (string, interface{}, error) {
-	algorithm := h.Get(SSEHeader)
-	if algorithm != SSEAlgorithmKMS {
-		return "", nil, ErrInvalidEncryptionMethod
-	}
-
-	contextStr, ok := h[SSEKmsContext]
-	if ok {
-		var context map[string]interface{}
-		var json = jsoniter.ConfigCompatibleWithStandardLibrary
-		if err := json.Unmarshal([]byte(contextStr[0]), &context); err != nil {
-			return "", nil, err
-		}
-		return h.Get(SSEKmsID), context, nil
-	}
-	return h.Get(SSEKmsID), nil, nil
-}
-
 var (
-	// SSEC represents AWS SSE-C. It provides functionality to handle
-	// SSE-C requests.
-	SSEC = ssec{}
-
 	// SSECopy represents AWS SSE-C for copy requests. It provides
 	// functionality to handle SSE-C copy requests.
 	SSECopy = ssecCopy{}
 )
 
-type ssec struct{}
 type ssecCopy struct{}
 
-// IsRequested returns true if the HTTP headers contains
-// at least one SSE-C header. SSE-C copy headers are ignored.
-func (ssec) IsRequested(h http.Header) bool {
-	if _, ok := h[SSECAlgorithm]; ok {
-		return true
-	}
-	if _, ok := h[SSECKey]; ok {
-		return true
-	}
-	if _, ok := h[SSECKeyMD5]; ok {
-		return true
-	}
-	return false
-}
-
 // IsRequested returns true if the HTTP headers contains
 // at least one SSE-C copy header. Regular SSE-C headers
 // are ignored.
 func (ssecCopy) IsRequested(h http.Header) bool {
-	if _, ok := h[SSECopyAlgorithm]; ok {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm]; ok {
 		return true
 	}
-	if _, ok := h[SSECopyKey]; ok {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerKey]; ok {
 		return true
 	}
-	if _, ok := h[SSECopyKeyMD5]; ok {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5]; ok {
 		return true
 	}
 	return false
 }
 
-// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
-// on success. SSE-C copy headers are ignored.
-func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
-	if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
-		return key, ErrInvalidCustomerAlgorithm
-	}
-	if h.Get(SSECKey) == "" {
-		return key, ErrMissingCustomerKey
-	}
-	if h.Get(SSECKeyMD5) == "" {
-		return key, ErrMissingCustomerKeyMD5
-	}
-
-	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECKey))
-	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
-		return key, ErrInvalidCustomerKey
-	}
-	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECKeyMD5))
-	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
-		return key, ErrCustomerKeyMD5Mismatch
-	}
-	copy(key[:], clientKey)
-	return key, nil
-}
-
 // ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key
 // on success. Regular SSE-C headers are ignored.
 func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) {
-	if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
+	if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != xhttp.AmzEncryptionAES {
 		return key, ErrInvalidCustomerAlgorithm
 	}
-	if h.Get(SSECopyKey) == "" {
+	if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey) == "" {
 		return key, ErrMissingCustomerKey
 	}
-	if h.Get(SSECopyKeyMD5) == "" {
+	if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5) == "" {
 		return key, ErrMissingCustomerKeyMD5
 	}
 
-	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKey))
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey))
 	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
 		return key, ErrInvalidCustomerKey
 	}
-	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKeyMD5))
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5))
 	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
 		return key, ErrCustomerKeyMD5Mismatch
 	}

cmd/crypto/header_test.go

@@ -18,26 +18,29 @@ import (
 	"net/http"
 	"sort"
 	"testing"
+
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 func TestIsRequested(t *testing.T) {
 	for i, test := range kmsIsRequestedTests {
-		if got := IsRequested(test.Header) && S3KMS.IsRequested(test.Header); got != test.Expected {
+		_, got := IsRequested(test.Header)
+		got = got && S3KMS.IsRequested(test.Header)
+		if got != test.Expected {
 			t.Errorf("SSE-KMS: Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
 	for i, test := range s3IsRequestedTests {
-		if got := IsRequested(test.Header) && S3.IsRequested(test.Header); got != test.Expected {
+		_, got := IsRequested(test.Header)
+		got = got && S3.IsRequested(test.Header)
+		if got != test.Expected {
 			t.Errorf("SSE-S3: Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
 	for i, test := range ssecIsRequestedTests {
-		if got := IsRequested(test.Header) && SSEC.IsRequested(test.Header); got != test.Expected {
-			t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
-		}
-	}
-	for i, test := range ssecCopyIsRequestedTests {
-		if got := IsRequested(test.Header) && SSECopy.IsRequested(test.Header); got != test.Expected {
+		_, got := IsRequested(test.Header)
+		got = got && SSEC.IsRequested(test.Header)
+		if got != test.Expected {
 			t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
@@ -131,11 +134,11 @@ var s3IsRequestedTests = []struct {
 	Header   http.Header
 	Expected bool
 }{
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},         // 0
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true},        // 1
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},               // 2
-	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},         // 3
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{SSEAlgorithmKMS}}, Expected: false}, // 4
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},                // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true},               // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},                      // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},                // 3
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{xhttp.AmzEncryptionKMS}}, Expected: false}, // 4
 }
 
 func TestS3IsRequested(t *testing.T) {
@@ -403,7 +406,7 @@ func TestSSECopyParse(t *testing.T) {
 		if err == nil && key == zeroKey {
 			t.Errorf("Test %d: parsed client key is zero key", i)
 		}
-		if _, ok := test.Header[SSECKey]; ok {
+		if _, ok := test.Header[xhttp.AmzServerSideEncryptionCustomerKey]; ok {
 			t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
 		}
 	}
@@ -414,47 +417,47 @@ var removeSensitiveHeadersTests = []struct {
 }{
 	{
 		Header: http.Header{
-			SSECKey:    []string{""},
-			SSECopyKey: []string{""},
+			xhttp.AmzServerSideEncryptionCustomerKey:     []string{""},
+			xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{""},
 		},
 		ExpectedHeader: http.Header{},
 	},
 	{ // Standard SSE-C request headers
 		Header: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
-			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 		ExpectedHeader: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 	},
 	{ // Standard SSE-C + SSE-C-copy request headers
 		Header: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
-			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm:  []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKey:        []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:     []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 		ExpectedHeader: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm:  []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:     []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 	},
 	{ // Standard SSE-C + metadata request headers
 		Header: http.Header{
-			SSECAlgorithm:       []string{SSEAlgorithmAES256},
-			SSECKey:             []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			"X-Amz-Meta-Test-1": []string{"Test-1"},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1":                            []string{"Test-1"},
 		},
 		ExpectedHeader: http.Header{
-			SSECAlgorithm:       []string{SSEAlgorithmAES256},
-			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			"X-Amz-Meta-Test-1": []string{"Test-1"},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1":                            []string{"Test-1"},
 		},
 	},
 	{ // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w

cmd/crypto/kes.go

@@ -177,11 +177,10 @@ func (kes *kesService) CreateKey(keyID string) error { return kes.client.CreateK
 // named key referenced by keyID. It also binds the generated key
 // cryptographically to the provided context.
 func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	var context bytes.Buffer
-	ctx.WriteTo(&context)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	var plainKey []byte
-	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context)
 	if err != nil {
 		return key, nil, err
 	}
@@ -200,11 +199,10 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	var context bytes.Buffer
-	ctx.WriteTo(&context)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	var plainKey []byte
-	plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context.Bytes())
+	plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context)
 	if err != nil {
 		return key, err
 	}
@@ -415,7 +413,7 @@ func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.
 		}
 
 		// If the error is not temp. / retryable => fail the request immediately.
-		if !xnet.IsNetworkOrHostDown(err) &&
+		if !xnet.IsNetworkOrHostDown(err, false) &&
 			!errors.Is(err, io.EOF) &&
 			!errors.Is(err, io.ErrUnexpectedEOF) &&
 			!errors.Is(err, context.DeadlineExceeded) {

cmd/crypto/key.go

@@ -103,7 +103,6 @@ func (key ObjectKey) Seal(extKey, iv [32]byte, domain, bucket, object string) Se
 func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucket, object string) error {
 	var (
 		unsealConfig sio.Config
-		decryptedKey bytes.Buffer
 	)
 	switch sealedKey.Algorithm {
 	default:
@@ -122,10 +121,9 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 		unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)}
 	}
 
-	if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
+	if out, err := sio.DecryptBuffer(key[:0], sealedKey.Key[:], unsealConfig); len(out) != 32 || err != nil {
 		return ErrSecretKeyMismatch
 	}
-	copy(key[:], decryptedKey.Bytes())
 	return nil
 }
 
@@ -165,11 +163,7 @@ func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) {
 	if !IsETagSealed(etag) {
 		return etag, nil
 	}
-	var buffer bytes.Buffer
 	mac := hmac.New(sha256.New, key[:])
 	mac.Write([]byte("SSE-etag"))
-	if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
-		return nil, err
-	}
-	return buffer.Bytes(), nil
+	return sio.DecryptBuffer(make([]byte, 0, len(etag)), etag, sio.Config{Key: mac.Sum(nil)})
 }

cmd/crypto/kms.go

@@ -39,6 +39,8 @@ type Context map[string]string
 //
 // WriteTo sorts the context keys and writes the sorted
 // key-value pairs as canonical JSON object to w.
+//
+// Note that neither keys nor values are escaped for JSON.
 func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	sortedKeys := make(sort.StringSlice, 0, len(c))
 	for k := range c {
@@ -67,6 +69,53 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	return n + int64(nn), err
 }
 
+// AppendTo appends the context in a canonical from to dst.
+//
+// AppendTo sorts the context keys and writes the sorted
+// key-value pairs as canonical JSON object to w.
+//
+// Note that neither keys nor values are escaped for JSON.
+func (c Context) AppendTo(dst []byte) (output []byte) {
+	if len(c) == 0 {
+		return append(dst, '{', '}')
+	}
+
+	// out should not escape.
+	out := bytes.NewBuffer(dst)
+
+	// No need to copy+sort
+	if len(c) == 1 {
+		for k, v := range c {
+			out.WriteString(`{"`)
+			out.WriteString(k)
+			out.WriteString(`":"`)
+			out.WriteString(v)
+			out.WriteString(`"}`)
+		}
+		return out.Bytes()
+	}
+
+	sortedKeys := make([]string, 0, len(c))
+	for k := range c {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Strings(sortedKeys)
+
+	out.WriteByte('{')
+	for i, k := range sortedKeys {
+		out.WriteByte('"')
+		out.WriteString(k)
+		out.WriteString(`":"`)
+		out.WriteString(c[k])
+		out.WriteByte('"')
+		if i < len(sortedKeys)-1 {
+			out.WriteByte(',')
+		}
+	}
+	out.WriteByte('}')
+	return out.Bytes()
+}
+
 // KMS represents an active and authenticted connection
 // to a Key-Management-Service. It supports generating
 // data key generation and unsealing of KMS-generated
@@ -155,13 +204,12 @@ func (kms *masterKeyKMS) Info() (info KMSInfo) {
 
 func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
 	var (
-		buffer     bytes.Buffer
 		derivedKey = kms.deriveKey(keyID, ctx)
 	)
-	if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 {
+	out, err := sio.DecryptBuffer(key[:0], sealedKey, sio.Config{Key: derivedKey[:]})
+	if err != nil || len(out) != 32 {
 		return key, err // TODO(aead): upgrade sio to use sio.Error
 	}
-	copy(key[:], buffer.Bytes())
 	return key, nil
 }
 
@@ -171,7 +219,7 @@ func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte)
 	}
 	mac := hmac.New(sha256.New, kms.masterKey[:])
 	mac.Write([]byte(keyID))
-	context.WriteTo(mac)
+	mac.Write(context.AppendTo(make([]byte, 0, 128)))
 	mac.Sum(key[:0])
 	return key
 }

cmd/crypto/kms_test.go

@@ -16,6 +16,7 @@ package crypto
 
 import (
 	"bytes"
+	"fmt"
 	"path"
 	"strings"
 	"testing"
@@ -83,3 +84,32 @@ func TestContextWriteTo(t *testing.T) {
 		}
 	}
 }
+
+func TestContextAppendTo(t *testing.T) {
+	for i, test := range contextWriteToTests {
+		dst := make([]byte, 0, 1024)
+		dst = test.Context.AppendTo(dst)
+		if s := string(dst); s != test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON)
+		}
+		// Append one more
+		dst = test.Context.AppendTo(dst)
+		if s := string(dst); s != test.ExpectedJSON+test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON+test.ExpectedJSON)
+		}
+	}
+}
+
+func BenchmarkContext_AppendTo(b *testing.B) {
+	tests := []Context{{}, {"bucket": "warp-benchmark-bucket"}, {"0": "1", "-": "2", ".": "#"}, {"34trg": "dfioutr89", "ikjfdghkjf": "jkedfhgfjkhg", "sdfhsdjkh": "if88889", "asddsirfh804": "kjfdshgdfuhgfg78-45604586#$%"}}
+	for _, test := range tests {
+		b.Run(fmt.Sprintf("%d-elems", len(test)), func(b *testing.B) {
+			dst := make([]byte, 0, 1024)
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				dst = test.AppendTo(dst[:0])
+			}
+		})
+	}
+}

cmd/crypto/metadata.go

@@ -15,19 +15,42 @@
 package crypto
 
 import (
-	"context"
-	"encoding/base64"
-	"errors"
-
 	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
+)
+
+const (
+	// MetaMultipart indicates that the object has been uploaded
+	// in multiple parts - via the S3 multipart API.
+	MetaMultipart = "X-Minio-Internal-Encrypted-Multipart"
+
+	// MetaIV is the random initialization vector (IV) used for
+	// the MinIO-internal key derivation.
+	MetaIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
+
+	// MetaAlgorithm is the algorithm used to derive internal keys
+	// and encrypt the objects.
+	MetaAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
+
+	// MetaSealedKeySSEC is the sealed object encryption key in case of SSE-C.
+	MetaSealedKeySSEC = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
+	// MetaSealedKeyS3 is the sealed object encryption key in case of SSE-S3
+	MetaSealedKeyS3 = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
+	// MetaSealedKeyKMS is the sealed object encryption key in case of SSE-KMS
+	MetaSealedKeyKMS = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key"
+
+	// MetaKeyID is the KMS master key ID used to generate/encrypt the data
+	// encryption key (DEK).
+	MetaKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
+	// MetaDataEncryptionKey is the sealed data encryption key (DEK) received from
+	// the KMS.
+	MetaDataEncryptionKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
 )
 
 // IsMultiPart returns true if the object metadata indicates
 // that it was uploaded using some form of server-side-encryption
 // and the S3 multipart API.
 func IsMultiPart(metadata map[string]string) bool {
-	if _, ok := metadata[SSEMultipart]; ok {
+	if _, ok := metadata[MetaMultipart]; ok {
 		return true
 	}
 	return false
@@ -37,8 +60,8 @@ func IsMultiPart(metadata map[string]string) bool {
 // information - e.g. the SSE-C key - from the metadata map.
 // It has the same semantics as RemoveSensitiveHeaders.
 func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
-	delete(metadata, SSECKey)
-	delete(metadata, SSECopyKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKey)
 	delete(metadata, xhttp.AmzMetaUnencryptedContentLength)
 	delete(metadata, xhttp.AmzMetaUnencryptedContentMD5)
 }
@@ -46,31 +69,36 @@ func RemoveSensitiveEntries(metadata map[string]string) { // The functions is te
 // RemoveSSEHeaders removes all crypto-specific SSE
 // header entries from the metadata map.
 func RemoveSSEHeaders(metadata map[string]string) {
-	delete(metadata, SSEHeader)
-	delete(metadata, SSEKmsID)
-	delete(metadata, SSEKmsContext)
-	delete(metadata, SSECKeyMD5)
-	delete(metadata, SSECAlgorithm)
+	delete(metadata, xhttp.AmzServerSideEncryption)
+	delete(metadata, xhttp.AmzServerSideEncryptionKmsID)
+	delete(metadata, xhttp.AmzServerSideEncryptionKmsContext)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerAlgorithm)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerKeyMD5)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5)
 }
 
 // RemoveInternalEntries removes all crypto-specific internal
 // metadata entries from the metadata map.
 func RemoveInternalEntries(metadata map[string]string) {
-	delete(metadata, SSEMultipart)
-	delete(metadata, SSEIV)
-	delete(metadata, SSESealAlgorithm)
-	delete(metadata, SSECSealedKey)
-	delete(metadata, S3SealedKey)
-	delete(metadata, S3KMSKeyID)
-	delete(metadata, S3KMSSealedKey)
+	delete(metadata, MetaMultipart)
+	delete(metadata, MetaAlgorithm)
+	delete(metadata, MetaIV)
+	delete(metadata, MetaSealedKeySSEC)
+	delete(metadata, MetaSealedKeyS3)
+	delete(metadata, MetaSealedKeyKMS)
+	delete(metadata, MetaKeyID)
+	delete(metadata, MetaDataEncryptionKey)
 }
 
 // IsSourceEncrypted returns true if the source is encrypted
 func IsSourceEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[SSECAlgorithm]; ok {
+	if _, ok := metadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm]; ok {
 		return true
 	}
-	if _, ok := metadata[SSEHeader]; ok {
+	if _, ok := metadata[xhttp.AmzServerSideEncryption]; ok {
 		return true
 	}
 	return false
@@ -82,10 +110,10 @@ func IsSourceEncrypted(metadata map[string]string) bool {
 // IsEncrypted only checks whether the metadata contains at least
 // one entry indicating SSE-C or SSE-S3.
 func IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[SSEIV]; ok {
+	if _, ok := metadata[MetaIV]; ok {
 		return true
 	}
-	if _, ok := metadata[SSESealAlgorithm]; ok {
+	if _, ok := metadata[MetaAlgorithm]; ok {
 		return true
 	}
 	if IsMultiPart(metadata) {
@@ -97,28 +125,7 @@ func IsEncrypted(metadata map[string]string) bool {
 	if SSEC.IsEncrypted(metadata) {
 		return true
 	}
-	return false
-}
-
-// IsEncrypted returns true if the object metadata indicates
-// that the object was uploaded using SSE-S3.
-func (s3) IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[S3SealedKey]; ok {
-		return true
-	}
-	if _, ok := metadata[S3KMSKeyID]; ok {
-		return true
-	}
-	if _, ok := metadata[S3KMSSealedKey]; ok {
-		return true
-	}
-	return false
-}
-
-// IsEncrypted returns true if the object metadata indicates
-// that the object was uploaded using SSE-C.
-func (ssec) IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[SSECSealedKey]; ok {
+	if S3KMS.IsEncrypted(metadata) {
 		return true
 	}
 	return false
@@ -129,157 +136,11 @@ func (ssec) IsEncrypted(metadata map[string]string) bool {
 // metadata is nil.
 func CreateMultipartMetadata(metadata map[string]string) map[string]string {
 	if metadata == nil {
-		return map[string]string{SSEMultipart: ""}
+		return map[string]string{MetaMultipart: ""}
 	}
-	metadata[SSEMultipart] = ""
+	metadata[MetaMultipart] = ""
 	return metadata
 }
 
-// CreateMetadata encodes the sealed object key into the metadata and returns
-// the modified metadata. If the keyID and the kmsKey is not empty it encodes
-// both into the metadata as well. It allocates a new metadata map if metadata
-// is nil.
-func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
-	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
-	}
-
-	// There are two possibilites:
-	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
-	// - We use a K/V -> There must be no key ID and no KMS data key.
-	// Otherwise, the caller has passed an invalid argument combination.
-	if keyID == "" && len(kmsKey) != 0 {
-		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
-	}
-	if keyID != "" && len(kmsKey) == 0 {
-		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
-	}
-
-	if metadata == nil {
-		metadata = make(map[string]string, 5)
-	}
-
-	metadata[SSESealAlgorithm] = sealedKey.Algorithm
-	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
-	metadata[S3SealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
-	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
-		metadata[S3KMSKeyID] = keyID
-		metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
-	}
-	return metadata
-}
-
-// ParseMetadata extracts all SSE-S3 related values from the object metadata
-// and checks whether they are well-formed. It returns the sealed object key
-// on success. If the metadata contains both, a KMS master key ID and a sealed
-// KMS data key it returns both. If the metadata does not contain neither a
-// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
-// KMS data key. Otherwise, it returns an error.
-func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
-	// Extract all required values from object metadata
-	b64IV, ok := metadata[SSEIV]
-	if !ok {
-		return keyID, kmsKey, sealedKey, errMissingInternalIV
-	}
-	algorithm, ok := metadata[SSESealAlgorithm]
-	if !ok {
-		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
-	}
-	b64SealedKey, ok := metadata[S3SealedKey]
-	if !ok {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
-	}
-
-	// There are two possibilites:
-	// - We use a KMS -> There must be a key ID and a KMS data key.
-	// - We use a K/V -> There must be no key ID and no KMS data key.
-	// Otherwise, the metadata is corrupted.
-	keyID, idPresent := metadata[S3KMSKeyID]
-	b64KMSSealedKey, kmsKeyPresent := metadata[S3KMSSealedKey]
-	if !idPresent && kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
-	}
-	if idPresent && !kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
-	}
-
-	// Check whether all extracted values are well-formed
-	iv, err := base64.StdEncoding.DecodeString(b64IV)
-	if err != nil || len(iv) != 32 {
-		return keyID, kmsKey, sealedKey, errInvalidInternalIV
-	}
-	if algorithm != SealAlgorithm {
-		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
-	}
-	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
-	if err != nil || len(encryptedKey) != 64 {
-		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
-	}
-	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
-		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
-		if err != nil {
-			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
-		}
-	}
-
-	sealedKey.Algorithm = algorithm
-	copy(sealedKey.IV[:], iv)
-	copy(sealedKey.Key[:], encryptedKey)
-	return keyID, kmsKey, sealedKey, nil
-}
-
-// CreateMetadata encodes the sealed key into the metadata and returns the modified metadata.
-// It allocates a new metadata map if metadata is nil.
-func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
-	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
-	}
-
-	if metadata == nil {
-		metadata = make(map[string]string, 3)
-	}
-	metadata[SSESealAlgorithm] = SealAlgorithm
-	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
-	metadata[SSECSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
-	return metadata
-}
-
-// ParseMetadata extracts all SSE-C related values from the object metadata
-// and checks whether they are well-formed. It returns the sealed object key
-// on success.
-func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
-	// Extract all required values from object metadata
-	b64IV, ok := metadata[SSEIV]
-	if !ok {
-		return sealedKey, errMissingInternalIV
-	}
-	algorithm, ok := metadata[SSESealAlgorithm]
-	if !ok {
-		return sealedKey, errMissingInternalSealAlgorithm
-	}
-	b64SealedKey, ok := metadata[SSECSealedKey]
-	if !ok {
-		return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
-	}
-
-	// Check whether all extracted values are well-formed
-	iv, err := base64.StdEncoding.DecodeString(b64IV)
-	if err != nil || len(iv) != 32 {
-		return sealedKey, errInvalidInternalIV
-	}
-	if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
-		return sealedKey, errInvalidInternalSealAlgorithm
-	}
-	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
-	if err != nil || len(encryptedKey) != 64 {
-		return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
-	}
-
-	sealedKey.Algorithm = algorithm
-	copy(sealedKey.IV[:], iv)
-	copy(sealedKey.Key[:], encryptedKey)
-	return sealedKey, nil
-}
-
 // IsETagSealed returns true if the etag seems to be encrypted.
 func IsETagSealed(etag []byte) bool { return len(etag) > 16 }

cmd/crypto/metadata_test.go

@@ -27,9 +27,9 @@ var isMultipartTests = []struct {
 	Metadata  map[string]string
 	Multipart bool
 }{
-	{Multipart: true, Metadata: map[string]string{SSEMultipart: ""}},                           // 0
+	{Multipart: true, Metadata: map[string]string{MetaMultipart: ""}},                          // 0
 	{Multipart: true, Metadata: map[string]string{"X-Minio-Internal-Encrypted-Multipart": ""}}, // 1
-	{Multipart: true, Metadata: map[string]string{SSEMultipart: "some-value"}},                 // 2
+	{Multipart: true, Metadata: map[string]string{MetaMultipart: "some-value"}},                // 2
 	{Multipart: false, Metadata: map[string]string{"": ""}},                                    // 3
 	{Multipart: false, Metadata: map[string]string{"X-Minio-Internal-EncryptedMultipart": ""}}, // 4
 }
@@ -46,13 +46,13 @@ var isEncryptedTests = []struct {
 	Metadata  map[string]string
 	Encrypted bool
 }{
-	{Encrypted: true, Metadata: map[string]string{SSEMultipart: ""}},                               // 0
-	{Encrypted: true, Metadata: map[string]string{SSEIV: ""}},                                      // 1
-	{Encrypted: true, Metadata: map[string]string{SSESealAlgorithm: ""}},                           // 2
-	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
-	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
-	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
-	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: true, Metadata: map[string]string{MetaMultipart: ""}},                              // 0
+	{Encrypted: true, Metadata: map[string]string{MetaIV: ""}},                                     // 1
+	{Encrypted: true, Metadata: map[string]string{MetaAlgorithm: ""}},                              // 2
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                          // 3
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}},                            // 4
+	{Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}},                                  // 5
+	{Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                      // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }
@@ -69,13 +69,13 @@ var s3IsEncryptedTests = []struct {
 	Metadata  map[string]string
 	Encrypted bool
 }{
-	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
-	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
-	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
-	{Encrypted: false, Metadata: map[string]string{SSECSealedKey: ""}},                             // 3
-	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
-	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
-	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: false, Metadata: map[string]string{MetaMultipart: ""}},                             // 0
+	{Encrypted: false, Metadata: map[string]string{MetaIV: ""}},                                    // 1
+	{Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}},                             // 2
+	{Encrypted: false, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                         // 3
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}},                            // 4
+	{Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}},                                  // 5
+	{Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                      // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }
@@ -92,13 +92,13 @@ var ssecIsEncryptedTests = []struct {
 	Metadata  map[string]string
 	Encrypted bool
 }{
-	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
-	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
-	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
-	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
-	{Encrypted: false, Metadata: map[string]string{S3SealedKey: ""}},                               // 4
-	{Encrypted: false, Metadata: map[string]string{S3KMSKeyID: ""}},                                // 5
-	{Encrypted: false, Metadata: map[string]string{S3KMSSealedKey: ""}},                            // 6
+	{Encrypted: false, Metadata: map[string]string{MetaMultipart: ""}},                             // 0
+	{Encrypted: false, Metadata: map[string]string{MetaIV: ""}},                                    // 1
+	{Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}},                             // 2
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                          // 3
+	{Encrypted: false, Metadata: map[string]string{MetaSealedKeyS3: ""}},                           // 4
+	{Encrypted: false, Metadata: map[string]string{MetaKeyID: ""}},                                 // 5
+	{Encrypted: false, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                     // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }
@@ -121,65 +121,65 @@ var s3ParseMetadataTests = []struct {
 }{
 	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}}, // 0
 	{
-		ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""},
+		ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{MetaIV: ""},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 1
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-S3"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 2
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSSealedKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaDataEncryptionKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 3
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 4
 	{
 		ExpectedErr: errInvalidInternalIV,
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: ""},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: "", MetaDataEncryptionKey: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 5
 	{
 		ExpectedErr: errInvalidInternalSealAlgorithm,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: "", MetaDataEncryptionKey: "",
 		},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 6
 	{
 		ExpectedErr: Errorf("The internal sealed key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
-			S3KMSKeyID: "", S3KMSSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, MetaSealedKeyS3: "",
+			MetaKeyID: "", MetaDataEncryptionKey: "",
 		},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 7
 	{
 		ExpectedErr: Errorf("The internal sealed KMS data key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
-			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
-			S3KMSSealedKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
+			MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), MetaKeyID: "key-1",
+			MetaDataEncryptionKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
 		},
 		DataKey: []byte{}, KeyID: "key-1", SealedKey: SealedKey{},
 	}, // 8
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
-			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "", S3KMSSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
+			MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), MetaKeyID: "", MetaDataEncryptionKey: "",
 		},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{Algorithm: SealAlgorithm},
 	}, // 9
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: SealAlgorithm,
-			S3SealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), S3KMSKeyID: "key-1",
-			S3KMSSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
+			MetaIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), MetaAlgorithm: SealAlgorithm,
+			MetaSealedKeyS3: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), MetaKeyID: "key-1",
+			MetaDataEncryptionKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
 		},
 		DataKey: make([]byte, 48), KeyID: "key-1", SealedKey: SealedKey{Algorithm: SealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
 	}, // 10
@@ -223,43 +223,43 @@ var ssecParseMetadataTests = []struct {
 
 	SealedKey SealedKey
 }{
-	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
-	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
+	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                      // 0
+	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{MetaIV: ""}, SealedKey: SealedKey{}}, // 1
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-C"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: ""}, SealedKey: SealedKey{},
 	}, // 2
 	{
 		ExpectedErr: errInvalidInternalIV,
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", SSECSealedKey: ""}, SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeySSEC: ""}, SealedKey: SealedKey{},
 	}, // 3
 	{
 		ExpectedErr: errInvalidInternalSealAlgorithm,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", SSECSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: "", MetaSealedKeySSEC: "",
 		},
 		SealedKey: SealedKey{},
 	}, // 4
 	{
 		ExpectedErr: Errorf("The internal sealed key for SSE-C is invalid"),
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, MetaSealedKeySSEC: "",
 		},
 		SealedKey: SealedKey{},
 	}, // 5
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
-			SSECSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
+			MetaSealedKeySSEC: base64.StdEncoding.EncodeToString(make([]byte, 64)),
 		},
 		SealedKey: SealedKey{Algorithm: SealAlgorithm},
 	}, // 6
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: InsecureSealAlgorithm,
-			SSECSealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
+			MetaIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), MetaAlgorithm: InsecureSealAlgorithm,
+			MetaSealedKeySSEC: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
 		},
 		SealedKey: SealedKey{Algorithm: InsecureSealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
 	}, // 7
@@ -267,8 +267,8 @@ var ssecParseMetadataTests = []struct {
 
 func TestCreateMultipartMetadata(t *testing.T) {
 	metadata := CreateMultipartMetadata(nil)
-	if v, ok := metadata[SSEMultipart]; !ok || v != "" {
-		t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", SSEMultipart, v, "")
+	if v, ok := metadata[MetaMultipart]; !ok || v != "" {
+		t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", MetaMultipart, v, "")
 	}
 }
 
@@ -411,20 +411,20 @@ var removeInternalEntriesTests = []struct {
 }{
 	{ // 0
 		Metadata: map[string]string{
-			SSEMultipart:     "",
-			SSEIV:            "",
-			SSESealAlgorithm: "",
-			SSECSealedKey:    "",
-			S3SealedKey:      "",
-			S3KMSKeyID:       "",
-			S3KMSSealedKey:   "",
+			MetaMultipart:         "",
+			MetaIV:                "",
+			MetaAlgorithm:         "",
+			MetaSealedKeySSEC:     "",
+			MetaSealedKeyS3:       "",
+			MetaKeyID:             "",
+			MetaDataEncryptionKey: "",
 		},
 		Expected: map[string]string{},
 	},
 	{ // 1
 		Metadata: map[string]string{
-			SSEMultipart:         "",
-			SSEIV:                "",
+			MetaMultipart:        "",
+			MetaIV:               "",
 			"X-Amz-Meta-A":       "X",
 			"X-Minio-Internal-B": "Y",
 		},

cmd/crypto/sse-c.go

@@ -0,0 +1,157 @@
+/*
+ * Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"net/http"
+
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+)
+
+type ssec struct{}
+
+var (
+	// SSEC represents AWS SSE-C. It provides functionality to handle
+	// SSE-C requests.
+	SSEC = ssec{}
+
+	_ Type = SSEC
+)
+
+// String returns the SSE domain as string. For SSE-C the
+// domain is "SSE-C".
+func (ssec) String() string { return "SSE-C" }
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C header. SSE-C copy headers are ignored.
+func (ssec) IsRequested(h http.Header) bool {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCustomerAlgorithm]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryptionCustomerKey]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryptionCustomerKeyMD5]; ok {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the metadata contains an SSE-C
+// entry inidicating that the object has been encrypted using
+// SSE-C.
+func (ssec) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[MetaSealedKeySSEC]; ok {
+		return true
+	}
+	return false
+}
+
+// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
+// on success. SSE-C copy headers are ignored.
+func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
+	if h.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm) != xhttp.AmzEncryptionAES {
+		return key, ErrInvalidCustomerAlgorithm
+	}
+	if h.Get(xhttp.AmzServerSideEncryptionCustomerKey) == "" {
+		return key, ErrMissingCustomerKey
+	}
+	if h.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5) == "" {
+		return key, ErrMissingCustomerKeyMD5
+	}
+
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCustomerKey))
+	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+		return key, ErrInvalidCustomerKey
+	}
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
+	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+		return key, ErrCustomerKeyMD5Mismatch
+	}
+	copy(key[:], clientKey)
+	return key, nil
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-C client key of the HTTP headers
+// and returns the decrypted object key.
+func (s3 ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := s3.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// CreateMetadata encodes the sealed key into the metadata
+// and returns the modified metadata. It allocates a new
+// metadata map if metadata is nil.
+func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
+	}
+
+	if metadata == nil {
+		metadata = make(map[string]string, 3)
+	}
+	metadata[MetaAlgorithm] = SealAlgorithm
+	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[MetaSealedKeySSEC] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-C related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success.
+func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[MetaIV]
+	if !ok {
+		return sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[MetaAlgorithm]
+	if !ok {
+		return sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[MetaSealedKeySSEC]
+	if !ok {
+		return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
+		return sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return sealedKey, nil
+}

cmd/crypto/sse-kms.go

@@ -0,0 +1,201 @@
+/*
+ * Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crypto
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"path"
+	"strings"
+
+	jsoniter "github.com/json-iterator/go"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+)
+
+type ssekms struct{}
+
+var (
+	// S3KMS represents AWS SSE-KMS. It provides functionality to
+	// handle SSE-KMS requests.
+	S3KMS = ssekms{}
+
+	_ Type = S3KMS
+)
+
+// String returns the SSE domain as string. For SSE-KMS the
+// domain is "SSE-KMS".
+func (ssekms) String() string { return "SSE-KMS" }
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-KMS header.
+func (ssekms) IsRequested(h http.Header) bool {
+	if _, ok := h[xhttp.AmzServerSideEncryptionKmsID]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryption]; ok {
+		return strings.ToUpper(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionAES // Return only true if the SSE header is specified and does not contain the SSE-S3 value
+	}
+	return false
+}
+
+// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
+// and the KMS context on success.
+func (ssekms) ParseHTTP(h http.Header) (string, Context, error) {
+	algorithm := h.Get(xhttp.AmzServerSideEncryption)
+	if algorithm != xhttp.AmzEncryptionKMS {
+		return "", nil, ErrInvalidEncryptionMethod
+	}
+
+	var ctx Context
+	if context, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
+		var json = jsoniter.ConfigCompatibleWithStandardLibrary
+		if err := json.Unmarshal([]byte(context[0]), &ctx); err != nil {
+			return "", nil, err
+		}
+	}
+	return h.Get(xhttp.AmzServerSideEncryptionKmsID), ctx, nil
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-KMS.
+func (ssekms) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[MetaSealedKeyKMS]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaKeyID]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaDataEncryptionKey]; ok {
+		return true
+	}
+	return false
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using KMS and returns the decrypted object
+// key.
+func (s3 ssekms) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata)
+	if err != nil {
+		return key, err
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return key, err
+	}
+	err = key.Unseal(unsealKey, sealedKey, s3.String(), bucket, object)
+	return key, err
+}
+
+// CreateMetadata encodes the sealed object key into the metadata and returns
+// the modified metadata. If the keyID and the kmsKey is not empty it encodes
+// both into the metadata as well. It allocates a new metadata map if metadata
+// is nil.
+func (ssekms) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the caller has passed an invalid argument combination.
+	if keyID == "" && len(kmsKey) != 0 {
+		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
+	}
+	if keyID != "" && len(kmsKey) == 0 {
+		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
+	}
+
+	if metadata == nil {
+		metadata = make(map[string]string, 5)
+	}
+
+	metadata[MetaAlgorithm] = sealedKey.Algorithm
+	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[MetaSealedKeyKMS] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
+		metadata[MetaKeyID] = keyID
+		metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	}
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-KMS related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success. If the metadata contains both, a KMS master key ID and a sealed
+// KMS data key it returns both. If the metadata does not contain neither a
+// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
+// KMS data key. Otherwise, it returns an error.
+func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[MetaIV]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[MetaAlgorithm]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[MetaSealedKeyKMS]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be a key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the metadata is corrupted.
+	keyID, idPresent := metadata[MetaKeyID]
+	b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
+	if !idPresent && kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
+	}
+	if idPresent && !kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm {
+		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
+	}
+	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
+		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+		if err != nil {
+			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
+		}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return keyID, kmsKey, sealedKey, nil
+}

cmd/crypto/sse-s3.go

@@ -0,0 +1,181 @@
+/*
+ * Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crypto
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"path"
+	"strings"
+
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+)
+
+type sses3 struct{}
+
+var (
+	// S3 represents AWS SSE-S3. It provides functionality to handle
+	// SSE-S3 requests.
+	S3 = sses3{}
+
+	_ Type = S3
+)
+
+// String returns the SSE domain as string. For SSE-S3 the
+// domain is "SSE-S3".
+func (sses3) String() string { return "SSE-S3" }
+
+func (sses3) IsRequested(h http.Header) bool {
+	_, ok := h[xhttp.AmzServerSideEncryption]
+	return ok && strings.ToLower(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionKMS // Return only true if the SSE header is specified and does not contain the SSE-KMS value
+}
+
+// ParseHTTP parses the SSE-S3 related HTTP headers and checks
+// whether they contain valid values.
+func (sses3) ParseHTTP(h http.Header) error {
+	if h.Get(xhttp.AmzServerSideEncryption) != xhttp.AmzEncryptionAES {
+		return ErrInvalidEncryptionMethod
+	}
+	return nil
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-S3.
+func (sses3) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[MetaSealedKeyS3]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaKeyID]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaDataEncryptionKey]; ok {
+		return true
+	}
+	return false
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using KMS and returns the decrypted object
+// key.
+func (s3 sses3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata)
+	if err != nil {
+		return key, err
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return key, err
+	}
+	err = key.Unseal(unsealKey, sealedKey, s3.String(), bucket, object)
+	return key, err
+}
+
+// CreateMetadata encodes the sealed object key into the metadata and returns
+// the modified metadata. If the keyID and the kmsKey is not empty it encodes
+// both into the metadata as well. It allocates a new metadata map if metadata
+// is nil.
+func (sses3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the caller has passed an invalid argument combination.
+	if keyID == "" && len(kmsKey) != 0 {
+		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
+	}
+	if keyID != "" && len(kmsKey) == 0 {
+		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
+	}
+
+	if metadata == nil {
+		metadata = make(map[string]string, 5)
+	}
+
+	metadata[MetaAlgorithm] = sealedKey.Algorithm
+	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[MetaSealedKeyS3] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
+		metadata[MetaKeyID] = keyID
+		metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	}
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-S3 related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success. If the metadata contains both, a KMS master key ID and a sealed
+// KMS data key it returns both. If the metadata does not contain neither a
+// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
+// KMS data key. Otherwise, it returns an error.
+func (sses3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[MetaIV]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[MetaAlgorithm]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[MetaSealedKeyS3]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be a key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the metadata is corrupted.
+	keyID, idPresent := metadata[MetaKeyID]
+	b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
+	if !idPresent && kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
+	}
+	if idPresent && !kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm {
+		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
+	}
+	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
+		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+		if err != nil {
+			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
+		}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return keyID, kmsKey, sealedKey, nil
+}

cmd/crypto/sse.go

@@ -17,44 +17,15 @@ package crypto
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
-	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/sio"
 )
 
-const (
-	// SSEMultipart is the metadata key indicating that the object
-	// was uploaded using the S3 multipart API and stored using
-	// some from of server-side-encryption.
-	SSEMultipart = "X-Minio-Internal-Encrypted-Multipart"
-
-	// SSEIV is the metadata key referencing the random initialization
-	// vector (IV) used for SSE-S3 and SSE-C key derivation.
-	SSEIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
-
-	// SSESealAlgorithm is the metadata key referencing the algorithm
-	// used by SSE-C and SSE-S3 to encrypt the object.
-	SSESealAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
-
-	// SSECSealedKey is the metadata key referencing the sealed object-key for SSE-C.
-	SSECSealedKey = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
-
-	// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
-	S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
-
-	// S3KMSKeyID is the metadata key referencing the KMS key-id used to
-	// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
-	S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
-
-	// S3KMSSealedKey is the metadata key referencing the encrypted key generated
-	// by KMS. It is only used for SSE-S3 + KMS.
-	S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
-)
-
 const (
 	// SealAlgorithm is the encryption/sealing algorithm used to derive & seal
 	// the key-encryption-key and to en/decrypt the object data.
@@ -67,39 +38,34 @@ const (
 	InsecureSealAlgorithm = "DARE-SHA256"
 )
 
-// String returns the SSE domain as string. For SSE-S3 the
-// domain is "SSE-S3".
-func (s3) String() string { return "SSE-S3" }
+// Type represents an AWS SSE type:
+//  • SSE-C
+//  • SSE-S3
+//  • SSE-KMS
+type Type interface {
+	fmt.Stringer
 
-// UnsealObjectKey extracts and decrypts the sealed object key
-// from the metadata using KMS and returns the decrypted object
-// key.
-func (sse s3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
-	keyID, kmsKey, sealedKey, err := sse.ParseMetadata(metadata)
-	if err != nil {
-		return
-	}
-	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
-	if err != nil {
-		return
-	}
-	err = key.Unseal(unsealKey, sealedKey, sse.String(), bucket, object)
-	return
+	IsRequested(http.Header) bool
+
+	IsEncrypted(map[string]string) bool
 }
 
-// String returns the SSE domain as string. For SSE-C the
-// domain is "SSE-C".
-func (ssec) String() string { return "SSE-C" }
-
-// UnsealObjectKey extracts and decrypts the sealed object key
-// from the metadata using the SSE-C client key of the HTTP headers
-// and returns the decrypted object key.
-func (sse ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
-	clientKey, err := sse.ParseHTTP(h)
-	if err != nil {
-		return
+// IsRequested returns true and the SSE Type if the HTTP headers
+// indicate that some form server-side encryption is requested.
+//
+// If no SSE headers are present then IsRequested returns false
+// and no Type.
+func IsRequested(h http.Header) (Type, bool) {
+	switch {
+	case S3.IsRequested(h):
+		return S3, true
+	case S3KMS.IsRequested(h):
+		return S3KMS, true
+	case SSEC.IsRequested(h):
+		return SSEC, true
+	default:
+		return nil, false
 	}
-	return unsealObjectKey(clientKey, metadata, bucket, object)
 }
 
 // UnsealObjectKey extracts and decrypts the sealed object key

cmd/crypto/vault.go

@@ -15,7 +15,6 @@
 package crypto
 
 import (
-	"bytes"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -224,11 +223,10 @@ func (v *vaultService) CreateKey(keyID string) error {
 // named key referenced by keyID. It also binds the generated key
 // cryptographically to the provided context.
 func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
-		"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context": base64.StdEncoding.EncodeToString(context),
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
 	if err != nil {
@@ -260,12 +258,11 @@ func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
-		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context":    base64.StdEncoding.EncodeToString(context),
 	}
 
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
@@ -294,12 +291,11 @@ func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (v *vaultService) UpdateKey(keyID string, sealedKey []byte, ctx Context) (rotatedKey []byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
-		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context":    base64.StdEncoding.EncodeToString(context),
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/rewrap/%s", keyID), payload)
 	if err != nil {

cmd/data-crawler.go

@@ -21,20 +21,20 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"math"
 	"math/rand"
 	"os"
 	"path"
-	"strconv"
 	"strings"
+	"sync"
 	"time"
 
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/config/heal"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/color"
-	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
@@ -43,8 +43,7 @@ import (
 
 const (
 	dataCrawlSleepPerFolder  = time.Millisecond // Time to wait between folders.
-	dataCrawlSleepDefMult    = 10.0             // Default multiplier for waits between operations.
-	dataCrawlStartDelay      = 5 * time.Minute  // Time to wait on startup and between cycles.
+	dataCrawlStartDelay      = 1 * time.Minute  // Time to wait on startup and between cycles.
 	dataUsageUpdateDirCycles = 16               // Visit all folders every n cycles.
 
 	healDeleteDangling    = true
@@ -53,15 +52,17 @@ const (
 )
 
 var (
-	globalHealConfig             heal.Config
+	globalHealConfig   heal.Config
+	globalHealConfigMu sync.Mutex
+
 	dataCrawlerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+	// Sleeper values are updated when config is loaded.
+	crawlerSleeper = newDynamicSleeper(10, 10*time.Second)
 )
 
-// initDataCrawler will start the crawler unless disabled.
+// initDataCrawler will start the crawler in the background.
 func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
-		go runDataCrawler(ctx, objAPI)
-	}
+	go runDataCrawler(ctx, objAPI)
 }
 
 // runDataCrawler will start a data crawler.
@@ -69,10 +70,10 @@ func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 // There should only ever be one crawler running per cluster.
 func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 	// Make sure only 1 crawler is running on the cluster.
-	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "runDataCrawler.lock")
+	locker := objAPI.NewNSLock(minioMetaBucket, "runDataCrawler.lock")
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	for {
-		err := locker.GetLock(dataCrawlerLeaderLockTimeout)
+		err := locker.GetLock(ctx, dataCrawlerLeaderLockTimeout)
 		if err != nil {
 			time.Sleep(time.Duration(r.Float64() * float64(dataCrawlStartDelay)))
 			continue
@@ -95,11 +96,21 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}
 
+	crawlTimer := time.NewTimer(dataCrawlStartDelay)
+	defer crawlTimer.Stop()
+
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-time.NewTimer(dataCrawlStartDelay).C:
+		case <-crawlTimer.C:
+			// Reset the timer for next cycle.
+			crawlTimer.Reset(dataCrawlStartDelay)
+
+			if intDataUpdateTracker.debug {
+				console.Debugln("starting crawler cycle")
+			}
+
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
@@ -141,27 +152,27 @@ type folderScanner struct {
 	newCache   dataUsageCache
 	withFilter *bloomFilter
 
-	dataUsageCrawlMult  float64
 	dataUsageCrawlDebug bool
 	healFolderInclude   uint32 // Include a clean folder one in n cycles.
 	healObjectSelect    uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 
 	newFolders      []cachedFolder
 	existingFolders []cachedFolder
+	disks           []StorageAPI
 }
 
 // crawlDataFolder will crawl the basepath+cache.Info.Name and return an updated cache.
 // The returned cache will always be valid, but may not be updated from the existing.
-// Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
+// Before each operation sleepDuration is called which can be used to temporarily halt the crawler.
 // If the supplied context is canceled the function will return at the first chance.
 func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
 	t := UTCNow()
 
 	logPrefix := color.Green("data-usage: ")
-	logSuffix := color.Blue(" - %v + %v", basePath, cache.Info.Name)
+	logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
 	if intDataUpdateTracker.debug {
 		defer func() {
-			logger.Info(logPrefix+" Crawl time: %v"+logSuffix, time.Since(t))
+			console.Debugf(logPrefix+" Crawl time: %v %s\n", time.Since(t), logSuffix)
 		}()
 
 	}
@@ -171,12 +182,6 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		return cache, errors.New("internal error: root scan attempted")
 	}
 
-	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		delayMult = dataCrawlSleepDefMult
-	}
-
 	s := folderScanner{
 		root:                basePath,
 		getSize:             getSize,
@@ -184,12 +189,23 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		newCache:            dataUsageCache{Info: cache.Info},
 		newFolders:          nil,
 		existingFolders:     nil,
-		dataUsageCrawlMult:  delayMult,
 		dataUsageCrawlDebug: intDataUpdateTracker.debug,
 		healFolderInclude:   0,
 		healObjectSelect:    0,
 	}
 
+	// Add disks for set healing.
+	if len(cache.Disks) > 0 {
+		objAPI, ok := newObjectLayerFn().(*erasureServerPools)
+		if ok {
+			s.disks = objAPI.GetDisksID(cache.Disks...)
+			if len(s.disks) != len(cache.Disks) {
+				console.Debugf(logPrefix+"Missing disks, want %d, found %d. Cannot heal. %s\n", len(cache.Disks), len(s.disks), logSuffix)
+				s.disks = s.disks[:0]
+			}
+		}
+	}
+
 	// Enable healing in XL mode.
 	if globalIsErasure {
 		// Include a clean folder one in n cycles.
@@ -199,28 +215,28 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	}
 	if len(cache.Info.BloomFilter) > 0 {
 		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
-		_, err := s.withFilter.ReadFrom(bytes.NewBuffer(cache.Info.BloomFilter))
+		_, err := s.withFilter.ReadFrom(bytes.NewReader(cache.Info.BloomFilter))
 		if err != nil {
 			logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
 			s.withFilter = nil
 		}
 	}
 	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Start crawling. Bloom filter: %v"+logSuffix, s.withFilter != nil)
+		console.Debugf(logPrefix+"Start crawling. Bloom filter: %v %s\n", s.withFilter != nil, logSuffix)
 	}
 
 	done := ctx.Done()
 	var flattenLevels = 2
 
 	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Cycle: %v, Entries: %v"+logSuffix, cache.Info.NextCycle, len(cache.Cache))
+		console.Debugf(logPrefix+"Cycle: %v, Entries: %v %s\n", cache.Info.NextCycle, len(cache.Cache), logSuffix)
 	}
 
 	// Always scan flattenLevels deep. Cache root is level 0.
 	todo := []cachedFolder{{name: cache.Info.Name, objectHealProbDiv: 1}}
 	for i := 0; i < flattenLevels; i++ {
 		if s.dataUsageCrawlDebug {
-			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
+			console.Debugf(logPrefix+"Level %v, scanning %v directories. %s\n", i, len(todo), logSuffix)
 		}
 		select {
 		case <-done:
@@ -236,7 +252,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	}
 
 	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
+		console.Debugf(logPrefix+"New folders: %v %s\n", s.newFolders, logSuffix)
 	}
 
 	// Add new folders first
@@ -252,7 +268,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 			continue
 		}
 		if du == nil {
-			logger.Info(logPrefix + "no disk usage provided" + logSuffix)
+			console.Debugln(logPrefix + "no disk usage provided" + logSuffix)
 			continue
 		}
 
@@ -265,7 +281,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	}
 
 	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
+		console.Debugf(logPrefix+"Existing folders: %v %s\n", len(s.existingFolders), logSuffix)
 	}
 
 	// Do selective scanning of existing folders.
@@ -292,13 +308,13 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 				if !s.withFilter.containsDir(folder.name) {
 					if !h.mod(s.oldCache.Info.NextCycle, s.healFolderInclude/folder.objectHealProbDiv) {
 						if s.dataUsageCrawlDebug {
-							logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder)
+							console.Debugf(logPrefix+"Skipping non-updated folder: %v %s\n", folder, logSuffix)
 						}
 						s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
 						continue
 					} else {
 						if s.dataUsageCrawlDebug {
-							logger.Info(logPrefix+"Adding non-updated folder to heal check: %v"+logSuffix, folder.name)
+							console.Debugf(logPrefix+"Adding non-updated folder to heal check: %v %s\n", folder.name, logSuffix)
 						}
 						// Update probability of including objects
 						folder.objectHealProbDiv = s.healFolderInclude
@@ -320,7 +336,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		s.newCache.replaceHashed(h, folder.parent, *du)
 	}
 	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Finished crawl, %v entries"+logSuffix, len(s.newCache.Cache))
+		console.Debugf(logPrefix+"Finished crawl, %v entries %s\n", len(s.newCache.Cache), logSuffix)
 	}
 	s.newCache.Info.LastUpdate = UTCNow()
 	s.newCache.Info.NextCycle++
@@ -334,6 +350,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
 	var nextFolders []cachedFolder
 	done := ctx.Done()
+	scannerLogPrefix := color.Green("folder-scanner:")
 	for _, folder := range folders {
 		select {
 		case <-done:
@@ -345,16 +362,14 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 
 		// If there are lifecycle rules for the prefix, remove the filter.
 		filter := f.withFilter
+		_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 		var activeLifeCycle *lifecycle.Lifecycle
-		if f.oldCache.Info.lifeCycle != nil {
-			_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
-			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
-				}
-				activeLifeCycle = f.oldCache.Info.lifeCycle
-				filter = nil
+		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
+			if f.dataUsageCrawlDebug {
+				console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
 			}
+			activeLifeCycle = f.oldCache.Info.lifeCycle
+			filter = nil
 		}
 		if _, ok := f.oldCache.Cache[thisHash.Key()]; filter != nil && ok {
 			// If folder isn't in filter and we have data, skip it completely.
@@ -362,19 +377,19 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				if !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
 					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					if f.dataUsageCrawlDebug {
-						logger.Info(color.Green("folder-scanner:")+" Skipping non-updated folder: %v", folder.name)
+						console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
 					}
 					continue
 				} else {
 					if f.dataUsageCrawlDebug {
-						logger.Info(color.Green("folder-scanner:")+" Adding non-updated folder to heal check: %v", folder.name)
+						console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
 					}
 					// If probability was already crawlerHealFolderInclude, keep it.
 					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
 		}
-		sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
+		crawlerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
 
 		cache := dataUsageEntry{}
 
@@ -384,14 +399,14 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
 			if bucket == "" {
 				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("folder-scanner:")+" no bucket (%s,%s)", f.root, entName)
+					console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName)
 				}
 				return nil
 			}
 
 			if isReservedOrInvalidBucket(bucket, false) {
 				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("folder-scanner:")+" invalid bucket: %v, entry: %v", bucket, entName)
+					console.Debugf(scannerLogPrefix+" invalid bucket: %v, entry: %v\n", bucket, entName)
 				}
 				return nil
 			}
@@ -408,8 +423,11 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				cache.addChildString(entName)
 
 				this := cachedFolder{name: entName, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv}
-				delete(existing, h.Key())
+
+				delete(existing, h.Key()) // h.Key() already accounted for.
+
 				cache.addChild(h)
+
 				if final {
 					if exists {
 						f.existingFolders = append(f.existingFolders, this)
@@ -423,7 +441,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			}
 
 			// Dynamic time delay.
-			t := UTCNow()
+			wait := crawlerSleeper.Timer(ctx)
 
 			// Get file size, ignore errors.
 			item := crawlItem{
@@ -434,18 +452,31 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				objectName: path.Base(entName),
 				debug:      f.dataUsageCrawlDebug,
 				lifeCycle:  activeLifeCycle,
-				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv),
+				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
-			size, err := f.getSize(item)
 
-			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			sizeSummary, err := f.getSize(item)
 			if err == errSkipFile {
+				wait() // wait to proceed to next entry.
+
 				return nil
 			}
-			logger.LogIf(ctx, err)
-			cache.Size += size
+
+			// successfully read means we have a valid object.
+
+			// Remove filename i.e is the meta file to construct object name
+			item.transformMetaDir()
+
+			// Object already accounted for, remove from heal map,
+			// simply because getSize() function already heals the
+			// object.
+			delete(existing, path.Join(item.bucket, item.objectPath()))
+
+			cache.addSizes(sizeSummary)
 			cache.Objects++
-			cache.ObjSizes.add(size)
+			cache.ObjSizes.add(sizeSummary.totalSize)
+
+			wait() // wait to proceed to next entry.
 
 			return nil
 		})
@@ -459,8 +490,8 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			continue
 		}
 
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
+		objAPI, ok := newObjectLayerFn().(*erasureServerPools)
+		if !ok || len(f.disks) == 0 {
 			continue
 		}
 
@@ -480,38 +511,139 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		// We therefore perform a heal check.
 		// If that doesn't bring it back we remove the folder and assume it was deleted.
 		// This means that the next run will not look for it.
+		// How to resolve results.
+		resolver := metadataResolutionParams{
+			dirQuorum: getReadQuorum(len(f.disks)),
+			objQuorum: getReadQuorum(len(f.disks)),
+			bucket:    "",
+		}
+
+		healObjectsPrefix := color.Green("healObjects:")
 		for k := range existing {
 			bucket, prefix := path2BucketObject(k)
 			if f.dataUsageCrawlDebug {
-				logger.Info(color.Green("folder-scanner:")+" checking disappeared folder: %v/%v", bucket, prefix)
+				console.Debugf(scannerLogPrefix+" checking disappeared folder: %v/%v\n", bucket, prefix)
 			}
 
 			// Dynamic time delay.
-			t := UTCNow()
+			wait := crawlerSleeper.Timer(ctx)
+			resolver.bucket = bucket
 
-			err = objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{Recursive: true, Remove: healDeleteDangling},
-				func(bucket, object, versionID string) error {
-					// Wait for each heal as per crawler frequency.
-					sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			foundObjs := false
+			dangling := false
+			ctx, cancel := context.WithCancel(ctx)
+			err := listPathRaw(ctx, listPathRawOptions{
+				disks:          f.disks,
+				bucket:         bucket,
+				path:           prefix,
+				recursive:      true,
+				reportNotFound: true,
+				minDisks:       len(f.disks), // We want full consistency.
+				// Weird, maybe transient error.
+				agreed: func(entry metaCacheEntry) {
+					if f.dataUsageCrawlDebug {
+						console.Debugf(healObjectsPrefix+" got agreement: %v\n", entry.name)
+					}
+				},
+				// Some disks have data for this.
+				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+					if f.dataUsageCrawlDebug {
+						console.Debugf(healObjectsPrefix+" got partial, %d agreed, errs: %v\n", nAgreed, errs)
+					}
 
-					defer func() {
-						t = UTCNow()
-					}()
-					return bgSeq.queueHealTask(healSource{
-						bucket:    bucket,
-						object:    object,
-						versionID: versionID,
-					}, madmin.HealItemObject)
-				})
+					// agreed value less than expected quorum
+					dangling = nAgreed < resolver.objQuorum || nAgreed < resolver.dirQuorum
 
-			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+					// Sleep and reset.
+					wait()
+					wait = crawlerSleeper.Timer(ctx)
+					entry, ok := entries.resolve(&resolver)
+					if !ok {
+						for _, err := range errs {
+							if err != nil {
+								return
+							}
+						}
 
-			if f.dataUsageCrawlDebug && err != nil {
-				logger.Info(color.Green("healObjects:")+" checking returned value %v", err)
+						// If no errors, queue it for healing.
+						entry, _ = entries.firstFound()
+					}
+
+					if f.dataUsageCrawlDebug {
+						console.Debugf(healObjectsPrefix+" resolved to: %v, dir: %v\n", entry.name, entry.isDir())
+					}
+
+					if entry.isDir() {
+						return
+					}
+					// We got an entry which we should be able to heal.
+					fiv, err := entry.fileInfoVersions(bucket)
+					if err != nil {
+						err := bgSeq.queueHealTask(healSource{
+							bucket:    bucket,
+							object:    entry.name,
+							versionID: "",
+						}, madmin.HealItemObject)
+						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+							logger.LogIf(ctx, err)
+						}
+						foundObjs = foundObjs || err == nil
+						return
+					}
+					for _, ver := range fiv.Versions {
+						// Sleep and reset.
+						wait()
+						wait = crawlerSleeper.Timer(ctx)
+						err := bgSeq.queueHealTask(healSource{
+							bucket:    bucket,
+							object:    fiv.Name,
+							versionID: ver.VersionID,
+						}, madmin.HealItemObject)
+						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+							logger.LogIf(ctx, err)
+						}
+						foundObjs = foundObjs || err == nil
+					}
+				},
+				// Too many disks failed.
+				finished: func(errs []error) {
+					if f.dataUsageCrawlDebug {
+						console.Debugf(healObjectsPrefix+" too many errors: %v\n", errs)
+					}
+					cancel()
+				},
+			})
+
+			if f.dataUsageCrawlDebug && err != nil && err != errFileNotFound {
+				console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err)
 			}
 
+			// If we found one or more disks with this folder, delete it.
+			if err == nil && dangling {
+				if f.dataUsageCrawlDebug {
+					console.Debugf(healObjectsPrefix+" deleting dangling directory %s\n", prefix)
+				}
+
+				objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{
+					Recursive: true,
+					Remove:    true,
+				},
+					func(bucket, object, versionID string) error {
+						// Wait for each heal as per crawler frequency.
+						wait()
+						wait = crawlerSleeper.Timer(ctx)
+						return bgSeq.queueHealTask(healSource{
+							bucket:    bucket,
+							object:    object,
+							versionID: versionID,
+						}, madmin.HealItemObject)
+					})
+			}
+
+			wait()
+
 			// Add unless healing returned an error.
-			if err == nil {
+			if foundObjs {
 				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv}
 				cache.addChild(hashPath(k))
 				if final {
@@ -535,6 +667,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 	var addDir func(entName string, typ os.FileMode) error
 	var dirStack = []string{f.root, folder.name}
 
+	deepScannerLogPrefix := color.Green("deep-scanner:")
 	addDir = func(entName string, typ os.FileMode) error {
 		select {
 		case <-done:
@@ -546,12 +679,12 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 			dirStack = append(dirStack, entName)
 			err := readDirFn(path.Join(dirStack...), addDir)
 			dirStack = dirStack[:len(dirStack)-1]
-			sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
+			crawlerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
 			return err
 		}
 
 		// Dynamic time delay.
-		t := UTCNow()
+		wait := crawlerSleeper.Timer(ctx)
 
 		// Get file size, ignore errors.
 		dirStack = append(dirStack, entName)
@@ -560,16 +693,14 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 
 		bucket, prefix := path2BucketObjectWithBasePath(f.root, fileName)
 		var activeLifeCycle *lifecycle.Lifecycle
-		if f.oldCache.Info.lifeCycle != nil {
-			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
-				}
-				activeLifeCycle = f.oldCache.Info.lifeCycle
+		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
+			if f.dataUsageCrawlDebug {
+				console.Debugf(deepScannerLogPrefix+" Prefix %q has active rules\n", prefix)
 			}
+			activeLifeCycle = f.oldCache.Info.lifeCycle
 		}
 
-		size, err := f.getSize(
+		sizeSummary, err := f.getSize(
 			crawlItem{
 				Path:       fileName,
 				Typ:        typ,
@@ -578,19 +709,24 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 				objectName: path.Base(entName),
 				debug:      f.dataUsageCrawlDebug,
 				lifeCycle:  activeLifeCycle,
-				heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv),
+				heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			})
 
-		// Don't sleep for really small amount of time
-		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
-
 		if err == errSkipFile {
+			// Wait to throttle IO
+			wait()
+
 			return nil
 		}
+
 		logger.LogIf(ctx, err)
-		cache.Size += size
+		cache.addSizes(sizeSummary)
 		cache.Objects++
-		cache.ObjSizes.add(size)
+		cache.ObjSizes.add(sizeSummary.totalSize)
+
+		// Wait to throttle IO
+		wait()
+
 		return nil
 	}
 	err := readDirFn(path.Join(dirStack...), addDir)
@@ -613,7 +749,15 @@ type crawlItem struct {
 	debug      bool
 }
 
-type getSizeFn func(item crawlItem) (int64, error)
+type sizeSummary struct {
+	totalSize      int64
+	replicatedSize int64
+	pendingSize    int64
+	failedSize     int64
+	replicaSize    int64
+}
+
+type getSizeFn func(item crawlItem) (sizeSummary, error)
 
 // transformMetaDir will transform a directory to prefix/file.ext
 func (i *crawlItem) transformMetaDir() {
@@ -632,6 +776,7 @@ type actionMeta struct {
 	oi               ObjectInfo
 	successorModTime time.Time // The modtime of the successor version
 	numVersions      int       // The number of versions of this object
+	bitRotScan       bool      // indicates if bitrot check was requested.
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -643,11 +788,20 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	if i.debug {
 		logger.LogIf(ctx, err)
 	}
+	applyActionsLogPrefix := color.Green("applyActions:")
 	if i.heal {
 		if i.debug {
-			logger.Info(color.Green("applyActions:")+" heal checking: %v/%v v%s", i.bucket, i.objectPath(), meta.oi.VersionID)
+			if meta.oi.VersionID != "" {
+				console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v v(%s)\n", i.bucket, i.objectPath(), meta.oi.VersionID)
+			} else {
+				console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v\n", i.bucket, i.objectPath())
+			}
 		}
-		res, err := o.HealObject(ctx, i.bucket, i.objectPath(), meta.oi.VersionID, madmin.HealOpts{Remove: healDeleteDangling})
+		healOpts := madmin.HealOpts{Remove: healDeleteDangling}
+		if meta.bitRotScan {
+			healOpts.ScanMode = madmin.HealDeepScan
+		}
+		res, err := o.HealObject(ctx, i.bucket, i.objectPath(), meta.oi.VersionID, healOpts)
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return 0
 		}
@@ -658,6 +812,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		size = res.ObjectSize
 	}
 	if i.lifeCycle == nil {
+		if i.debug {
+			console.Debugf(applyActionsLogPrefix+" no lifecycle rules to apply: %q\n", i.objectPath())
+		}
 		return size
 	}
 
@@ -672,14 +829,26 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			IsLatest:         meta.oi.IsLatest,
 			NumVersions:      meta.numVersions,
 			SuccessorModTime: meta.successorModTime,
+			RestoreOngoing:   meta.oi.RestoreOngoing,
+			RestoreExpires:   meta.oi.RestoreExpires,
+			TransitionStatus: meta.oi.TransitionStatus,
 		})
 	if i.debug {
-		logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action)
+		if versionID != "" {
+			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, action)
+		} else {
+			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
+		}
 	}
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
+	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
 	default:
 		// No action.
+		if i.debug {
+			console.Debugf(applyActionsLogPrefix+" object not expirable: %q\n", i.objectPath())
+		}
 		return size
 	}
 
@@ -706,22 +875,27 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	size = obj.Size
 
 	// Recalculate action.
-	action = i.lifeCycle.ComputeAction(
-		lifecycle.ObjectOpts{
-			Name:             i.objectPath(),
-			UserTags:         obj.UserTags,
-			ModTime:          obj.ModTime,
-			VersionID:        obj.VersionID,
-			DeleteMarker:     obj.DeleteMarker,
-			IsLatest:         obj.IsLatest,
-			NumVersions:      meta.numVersions,
-			SuccessorModTime: meta.successorModTime,
-		})
+	lcOpts := lifecycle.ObjectOpts{
+		Name:             i.objectPath(),
+		UserTags:         obj.UserTags,
+		ModTime:          obj.ModTime,
+		VersionID:        obj.VersionID,
+		DeleteMarker:     obj.DeleteMarker,
+		IsLatest:         obj.IsLatest,
+		NumVersions:      meta.numVersions,
+		SuccessorModTime: meta.successorModTime,
+		RestoreOngoing:   obj.RestoreOngoing,
+		RestoreExpires:   obj.RestoreExpires,
+		TransitionStatus: obj.TransitionStatus,
+	}
+	action = i.lifeCycle.ComputeAction(lcOpts)
 	if i.debug {
-		logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
+		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
 	}
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
+	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
 	default:
 		// No action.
 		return size
@@ -729,7 +903,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 
 	opts := ObjectOptions{}
 	switch action {
-	case lifecycle.DeleteVersionAction:
+	case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
 		// Defensive code, should never happen
 		if obj.VersionID == "" {
 			return size
@@ -738,14 +912,40 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			locked := enforceRetentionForDeletion(ctx, obj)
 			if locked {
 				if i.debug {
-					logger.Info(color.Green("applyActions:")+" lifecycle: %s is locked, not deleting", i.objectPath())
+					if obj.VersionID != "" {
+						console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", i.objectPath(), obj.VersionID)
+					} else {
+						console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", i.objectPath())
+					}
 				}
 				return size
 			}
 		}
 		opts.VersionID = obj.VersionID
-	case lifecycle.DeleteAction:
+	case lifecycle.DeleteAction, lifecycle.DeleteRestoredAction:
 		opts.Versioned = globalBucketVersioningSys.Enabled(i.bucket)
+	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+		if obj.TransitionStatus == "" {
+			opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
+			opts.VersionID = obj.VersionID
+			opts.TransitionStatus = lifecycle.TransitionPending
+			if _, err = o.DeleteObject(ctx, obj.Bucket, obj.Name, opts); err != nil {
+				// Assume it is still there.
+				logger.LogIf(ctx, err)
+				return size
+			}
+		}
+		globalTransitionState.queueTransitionTask(obj)
+		return 0
+	}
+
+	if obj.TransitionStatus != "" {
+		if err := deleteTransitionedObject(ctx, o, i.bucket, i.objectPath(), lcOpts, action, false); err != nil {
+			logger.LogIf(ctx, err)
+			return size
+		}
+		// Notification already sent at *deleteTransitionedObject*, return '0' here.
+		return 0
 	}
 
 	obj, err = o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
@@ -775,22 +975,176 @@ func (i *crawlItem) objectPath() string {
 	return path.Join(i.prefix, i.objectName)
 }
 
-// sleepDuration multiplies the duration d by x and sleeps if is more than 100 micro seconds.
-// sleep is limited to max 1 second.
-func sleepDuration(d time.Duration, x float64) {
-	// Don't sleep for really small amount of time
-	if d := time.Duration(float64(d) * x); d > time.Microsecond*100 {
-		if d > time.Second {
-			d = time.Second
+// healReplication will heal a scanned item that has failed replication.
+func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
+	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
+		// heal delete marker replication failure or versioned delete replication failure
+		if oi.ReplicationStatus == replication.Pending ||
+			oi.ReplicationStatus == replication.Failed ||
+			oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
+			i.healReplicationDeletes(ctx, o, oi)
+			return
 		}
-		time.Sleep(d)
+	}
+	switch oi.ReplicationStatus {
+	case replication.Pending:
+		sizeS.pendingSize += oi.Size
+		globalReplicationState.queueReplicaTask(oi)
+	case replication.Failed:
+		sizeS.failedSize += oi.Size
+		globalReplicationState.queueReplicaTask(oi)
+	case replication.Completed, "COMPLETE":
+		sizeS.replicatedSize += oi.Size
+	case replication.Replica:
+		sizeS.replicaSize += oi.Size
 	}
 }
 
-// healReplication will heal a scanned item that has failed replication.
-func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta actionMeta) {
-	if meta.oi.ReplicationStatus == replication.Pending ||
-		meta.oi.ReplicationStatus == replication.Failed {
-		globalReplicationState.queueReplicaTask(meta.oi)
+// healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
+func (i *crawlItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
+	// handle soft delete and permanent delete failures here.
+	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
+		versionID := ""
+		dmVersionID := ""
+		if oi.VersionPurgeStatus.Empty() {
+			dmVersionID = oi.VersionID
+		} else {
+			versionID = oi.VersionID
+		}
+		globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
+			DeletedObject: DeletedObject{
+				ObjectName:                    oi.Name,
+				DeleteMarkerVersionID:         dmVersionID,
+				VersionID:                     versionID,
+				DeleteMarkerReplicationStatus: string(oi.ReplicationStatus),
+				DeleteMarkerMTime:             DeleteMarkerMTime{oi.ModTime},
+				DeleteMarker:                  oi.DeleteMarker,
+				VersionPurgeStatus:            oi.VersionPurgeStatus,
+			},
+			Bucket: oi.Bucket,
+		})
 	}
 }
+
+type dynamicSleeper struct {
+	mu sync.RWMutex
+
+	// Sleep factor
+	factor float64
+
+	// maximum sleep cap,
+	// set to <= 0 to disable.
+	maxSleep time.Duration
+
+	// Don't sleep at all, if time taken is below this value.
+	// This is to avoid too small costly sleeps.
+	minSleep time.Duration
+
+	// cycle will be closed
+	cycle chan struct{}
+}
+
+// newDynamicSleeper
+func newDynamicSleeper(factor float64, maxWait time.Duration) *dynamicSleeper {
+	return &dynamicSleeper{
+		factor:   factor,
+		cycle:    make(chan struct{}),
+		maxSleep: maxWait,
+		minSleep: 100 * time.Microsecond,
+	}
+}
+
+// Timer returns a timer that has started.
+// When the returned function is called it will wait.
+func (d *dynamicSleeper) Timer(ctx context.Context) func() {
+	t := time.Now()
+	return func() {
+		doneAt := time.Now()
+		for {
+			// Grab current values
+			d.mu.RLock()
+			minWait, maxWait := d.minSleep, d.maxSleep
+			factor := d.factor
+			cycle := d.cycle
+			d.mu.RUnlock()
+			elapsed := doneAt.Sub(t)
+			// Don't sleep for really small amount of time
+			wantSleep := time.Duration(float64(elapsed) * factor)
+			if wantSleep <= minWait {
+				return
+			}
+			if maxWait > 0 && wantSleep > maxWait {
+				wantSleep = maxWait
+			}
+			timer := time.NewTimer(wantSleep)
+			select {
+			case <-ctx.Done():
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return
+			case <-timer.C:
+				return
+			case <-cycle:
+				if !timer.Stop() {
+					// We expired.
+					<-timer.C
+					return
+				}
+			}
+		}
+	}
+}
+
+// Sleep sleeps the specified time multiplied by the sleep factor.
+// If the factor is updated the sleep will be done again with the new factor.
+func (d *dynamicSleeper) Sleep(ctx context.Context, base time.Duration) {
+	for {
+		// Grab current values
+		d.mu.RLock()
+		minWait, maxWait := d.minSleep, d.maxSleep
+		factor := d.factor
+		cycle := d.cycle
+		d.mu.RUnlock()
+		// Don't sleep for really small amount of time
+		wantSleep := time.Duration(float64(base) * factor)
+		if wantSleep <= minWait {
+			return
+		}
+		if maxWait > 0 && wantSleep > maxWait {
+			wantSleep = maxWait
+		}
+		timer := time.NewTimer(wantSleep)
+		select {
+		case <-ctx.Done():
+			if !timer.Stop() {
+				<-timer.C
+			}
+			return
+		case <-timer.C:
+			return
+		case <-cycle:
+			if !timer.Stop() {
+				// We expired.
+				<-timer.C
+				return
+			}
+		}
+	}
+}
+
+// Update the current settings and cycle all waiting.
+// Parameters are the same as in the contructor.
+func (d *dynamicSleeper) Update(factor float64, maxWait time.Duration) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if math.Abs(d.factor-factor) < 1e-10 && d.maxSleep == maxWait {
+		return nil
+	}
+	// Update values and cycle waiting.
+	close(d.cycle)
+	d.factor = factor
+	d.maxSleep = maxWait
+	d.cycle = make(chan struct{})
+	return nil
+}

cmd/data-update-tracker.go

@@ -34,19 +34,20 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/env"
 	"github.com/willf/bloom"
 )
 
 const (
 	// Estimate bloom filter size. With this many items
-	dataUpdateTrackerEstItems = 1000000
+	dataUpdateTrackerEstItems = 10000000
 	// ... we want this false positive rate:
 	dataUpdateTrackerFP        = 0.99
-	dataUpdateTrackerQueueSize = 10000
+	dataUpdateTrackerQueueSize = 0
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 3
+	dataUpdateTrackerVersion      = 4
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 )
 
@@ -79,7 +80,7 @@ func newDataUpdateTracker() *dataUpdateTracker {
 		Current: dataUpdateFilter{
 			idx: 1,
 		},
-		debug:      env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn,
+		debug:      env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn || serverDebugLog,
 		input:      make(chan string, dataUpdateTrackerQueueSize),
 		save:       make(chan struct{}, 1),
 		saveExited: make(chan struct{}),
@@ -168,6 +169,44 @@ func (d *dataUpdateTracker) current() uint64 {
 	return d.Current.idx
 }
 
+// latestWithDir returns the highest index that contains the directory.
+// This means that any cycle higher than this does NOT contain the entry.
+func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
+	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
+	bucket, _ := path2BucketObjectWithBasePath("", dir)
+	if bucket == "" {
+		if d.debug && len(dir) > 0 {
+			console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", dir)
+		}
+		return d.current()
+	}
+	if isReservedOrInvalidBucket(bucket, false) {
+		if d.debug {
+			console.Debugf(dateUpdateTrackerLogPrefix+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, dir)
+		}
+		return d.current()
+	}
+
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if d.Current.bf.containsDir(dir) || d.Current.idx == 0 {
+		return d.Current.idx
+	}
+	if d.debug {
+		console.Debugf(dateUpdateTrackerLogPrefix+" current bloom does NOT contains dir %s\n", dir)
+	}
+
+	idx := d.Current.idx - 1
+	for {
+		f := d.History.find(idx)
+		if f == nil || f.bf.containsDir(dir) || idx == 0 {
+			break
+		}
+		idx--
+	}
+	return idx
+}
+
 // start will load the current data from the drives start collecting information and
 // start a saver goroutine.
 // All of these will exit when the context is canceled.
@@ -198,7 +237,7 @@ func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
 		cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
 		f, err := os.Open(cacheFormatPath)
 		if err != nil {
-			if os.IsNotExist(err) {
+			if osIsNotExist(err) {
 				continue
 			}
 			logger.LogIf(ctx, err)
@@ -244,7 +283,7 @@ func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Durati
 		d.Saved = UTCNow()
 		err := d.serialize(&buf)
 		if d.debug {
-			logger.Info(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v", buf.Len(), d.Current.idx)
+			console.Debugf(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v\n", buf.Len(), d.Current.idx)
 		}
 		d.dirty = false
 		d.mu.Unlock()
@@ -263,7 +302,7 @@ func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Durati
 			cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
 			err := ioutil.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
 			if err != nil {
-				if os.IsNotExist(err) {
+				if osIsNotExist(err) {
 					continue
 				}
 				logger.LogIf(ctx, err)
@@ -366,8 +405,8 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
-	case 1, 2:
-		logger.Info(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
+	case 1, 2, 3:
+		console.Println(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
 		return nil
 	case dataUpdateTrackerVersion:
 	default:
@@ -440,41 +479,67 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 // start a collector that picks up entries from objectUpdatedCh
 // and adds them  to the current bloom filter.
 func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case in := <-d.input:
-			bucket, _ := path2BucketObjectWithBasePath("", in)
-			if bucket == "" {
-				if d.debug && len(in) > 0 {
-					logger.Info(color.Green("data-usage:")+" no bucket (%s)", in)
-				}
-				continue
+	for in := range d.input {
+		bucket, _ := path2BucketObjectWithBasePath("", in)
+		if bucket == "" {
+			if d.debug && len(in) > 0 {
+				console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
 			}
-
-			if isReservedOrInvalidBucket(bucket, false) {
-				if false && d.debug {
-					logger.Info(color.Green("data-usage:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
-				}
-				continue
-			}
-			split := splitPathDeterministic(in)
-
-			// Add all paths until level 3.
-			d.mu.Lock()
-			for i := range split {
-				if d.debug && false {
-					logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
-				}
-				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
-			}
-			d.dirty = d.dirty || len(split) > 0
-			d.mu.Unlock()
+			continue
 		}
+
+		if isReservedOrInvalidBucket(bucket, false) {
+			if d.debug {
+				console.Debugf(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, in)
+			}
+			continue
+		}
+		split := splitPathDeterministic(in)
+
+		// Add all paths until done.
+		d.mu.Lock()
+		for i := range split {
+			if d.debug {
+				console.Debugln(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+			}
+			d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+		}
+		d.dirty = d.dirty || len(split) > 0
+		d.mu.Unlock()
 	}
 }
 
+// markDirty adds the supplied path to the current bloom filter.
+func (d *dataUpdateTracker) markDirty(in string) {
+	bucket, _ := path2BucketObjectWithBasePath("", in)
+	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
+	if bucket == "" {
+		if d.debug && len(in) > 0 {
+			console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", in)
+		}
+		return
+	}
+
+	if isReservedOrInvalidBucket(bucket, false) {
+		if d.debug && false {
+			console.Debugf(dateUpdateTrackerLogPrefix+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, in)
+		}
+		return
+	}
+	split := splitPathDeterministic(in)
+
+	// Add all paths until done.
+	d.mu.Lock()
+	for i := range split {
+		if d.debug {
+			console.Debugln(dateUpdateTrackerLogPrefix + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+		}
+		d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+	}
+	d.dirty = d.dirty || len(split) > 0
+	d.mu.Unlock()
+}
+
 // find entry with specified index.
 // Returns nil if not found.
 func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
@@ -534,8 +599,13 @@ func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint6
 // cycleFilter will cycle the bloom filter to start recording to index y if not already.
 // The response will contain a bloom filter starting at index x up to, but not including index y.
 // If y is 0, the response will not update y, but return the currently recorded information
-// from the up until and including current y.
-func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
+// from the oldest (unless 0, then it will be all) until and including current y.
+func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
+	if req.OldestClean != "" {
+		return &bloomFilterResponse{OldestIdx: d.latestWithDir(req.OldestClean)}, nil
+	}
+	current := req.Current
+	oldest := req.Oldest
 	d.mu.Lock()
 	defer d.mu.Unlock()
 	if current == 0 {
@@ -543,14 +613,17 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uin
 			return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
 		}
 		d.History.sort()
-		return d.filterFrom(ctx, d.History[len(d.History)-1].idx, d.Current.idx), nil
+		if oldest == 0 {
+			oldest = d.History[len(d.History)-1].idx
+		}
+		return d.filterFrom(ctx, oldest, d.Current.idx), nil
 	}
 
 	// Move current to history if new one requested
 	if d.Current.idx != current {
 		d.dirty = true
 		if d.debug {
-			logger.Info(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v", d.Current.idx, current)
+			console.Debugf(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v\n", d.Current.idx, current)
 		}
 
 		d.History = append(d.History, d.Current)
@@ -571,7 +644,7 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uin
 // Trailing slashes are removed.
 // Returns 0 length if no parts are found after trimming.
 func splitPathDeterministic(in string) []string {
-	split := strings.Split(in, SlashSeparator)
+	split := strings.Split(decodeDirObject(in), SlashSeparator)
 
 	// Trim empty start/end
 	for len(split) > 0 {
@@ -587,10 +660,6 @@ func splitPathDeterministic(in string) []string {
 		split = split[:len(split)-1]
 	}
 
-	// Return up to 3 parts.
-	if len(split) > 3 {
-		split = split[:3]
-	}
 	return split
 }
 
@@ -599,6 +668,9 @@ func splitPathDeterministic(in string) []string {
 type bloomFilterRequest struct {
 	Oldest  uint64
 	Current uint64
+	// If set the oldest clean version will be returned in OldestIdx
+	// and the rest of the request will be ignored.
+	OldestClean string
 }
 
 type bloomFilterResponse struct {
@@ -615,10 +687,9 @@ type bloomFilterResponse struct {
 }
 
 // ObjectPathUpdated indicates a path has been updated.
-// The function will never block.
+// The function will block until the entry has been picked up.
 func ObjectPathUpdated(s string) {
-	select {
-	case objectUpdatedCh <- s:
-	default:
+	if intDataUpdateTracker != nil {
+		intDataUpdateTracker.markDirty(s)
 	}
 }

cmd/data-update-tracker_test.go

@@ -169,7 +169,12 @@ func TestDataUpdateTracker(t *testing.T) {
 		})
 	}
 	// Cycle to history
-	_, err = dut.cycleFilter(ctx, 1, 2)
+	req := bloomFilterRequest{
+		Oldest:  1,
+		Current: 2,
+	}
+
+	_, err = dut.cycleFilter(ctx, req)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -200,7 +205,11 @@ func TestDataUpdateTracker(t *testing.T) {
 	if dut.current() != 2 {
 		t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
 	}
-	bfr2, err := dut.cycleFilter(ctx, 1, 3)
+	req = bloomFilterRequest{
+		Oldest:  1,
+		Current: 3,
+	}
+	bfr2, err := dut.cycleFilter(ctx, req)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -216,7 +225,7 @@ func TestDataUpdateTracker(t *testing.T) {
 
 	// Rerun test with returned bfr2
 	bf := dut.newBloomFilter()
-	_, err = bf.ReadFrom(bytes.NewBuffer(bfr2.Filter))
+	_, err = bf.ReadFrom(bytes.NewReader(bfr2.Filter))
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/data-usage-cache.go

@@ -17,11 +17,11 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"path"
 	"path/filepath"
 	"strings"
@@ -45,20 +45,40 @@ type sizeHistogram [dataUsageBucketLen]uint64
 
 //msgp:tuple dataUsageEntry
 type dataUsageEntry struct {
+	// These fields do no include any children.
+	Size                   int64
+	ReplicatedSize         uint64
+	ReplicationPendingSize uint64
+	ReplicationFailedSize  uint64
+	ReplicaSize            uint64
+	Objects                uint64
+	ObjSizes               sizeHistogram
+	Children               dataUsageHashMap
+}
+
+//msgp:tuple dataUsageEntryV2
+type dataUsageEntryV2 struct {
 	// These fields do no include any children.
 	Size     int64
 	Objects  uint64
 	ObjSizes sizeHistogram
-
 	Children dataUsageHashMap
 }
 
-// dataUsageCache contains a cache of data usage entries.
+// dataUsageCache contains a cache of data usage entries latest version 3.
 type dataUsageCache struct {
 	Info  dataUsageCacheInfo
+	Disks []string
 	Cache map[string]dataUsageEntry
 }
 
+// dataUsageCache contains a cache of data usage entries version 2.
+type dataUsageCacheV2 struct {
+	Info  dataUsageCacheInfo
+	Disks []string
+	Cache map[string]dataUsageEntryV2
+}
+
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -75,10 +95,23 @@ type dataUsageCacheInfo struct {
 	lifeCycle   *lifecycle.Lifecycle `msg:"-"`
 }
 
+func (e *dataUsageEntry) addSizes(summary sizeSummary) {
+	e.Size += summary.totalSize
+	e.ReplicatedSize += uint64(summary.replicatedSize)
+	e.ReplicationFailedSize += uint64(summary.failedSize)
+	e.ReplicationPendingSize += uint64(summary.pendingSize)
+	e.ReplicaSize += uint64(summary.replicaSize)
+}
+
 // merge other data usage entry into this, excluding children.
 func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Objects += other.Objects
 	e.Size += other.Size
+	e.ReplicationPendingSize += other.ReplicationPendingSize
+	e.ReplicationFailedSize += other.ReplicationFailedSize
+	e.ReplicatedSize += other.ReplicatedSize
+	e.ReplicaSize += other.ReplicaSize
+
 	for i, v := range other.ObjSizes[:] {
 		e.ObjSizes[i] += v
 	}
@@ -212,11 +245,15 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	}
 	flat := d.flatten(*e)
 	return DataUsageInfo{
-		LastUpdate:        d.Info.LastUpdate,
-		ObjectsTotalCount: flat.Objects,
-		ObjectsTotalSize:  uint64(flat.Size),
-		BucketsCount:      uint64(len(e.Children)),
-		BucketsUsage:      d.bucketsUsageInfo(buckets),
+		LastUpdate:             d.Info.LastUpdate,
+		ObjectsTotalCount:      flat.Objects,
+		ObjectsTotalSize:       uint64(flat.Size),
+		ReplicatedSize:         flat.ReplicatedSize,
+		ReplicationFailedSize:  flat.ReplicationFailedSize,
+		ReplicationPendingSize: flat.ReplicationPendingSize,
+		ReplicaSize:            flat.ReplicaSize,
+		BucketsCount:           uint64(len(e.Children)),
+		BucketsUsage:           d.bucketsUsageInfo(buckets),
 	}
 }
 
@@ -342,9 +379,13 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		}
 		flat := d.flatten(*e)
 		dst[bucket.Name] = BucketUsageInfo{
-			Size:                 uint64(flat.Size),
-			ObjectsCount:         flat.Objects,
-			ObjectSizesHistogram: flat.ObjSizes.toMap(),
+			Size:                   uint64(flat.Size),
+			ObjectsCount:           flat.Objects,
+			ReplicationPendingSize: flat.ReplicationPendingSize,
+			ReplicatedSize:         flat.ReplicatedSize,
+			ReplicationFailedSize:  flat.ReplicationFailedSize,
+			ReplicaSize:            flat.ReplicaSize,
+			ObjectSizesHistogram:   flat.ObjSizes.toMap(),
 		}
 	}
 	return dst
@@ -359,9 +400,13 @@ func (d *dataUsageCache) bucketUsageInfo(bucket string) BucketUsageInfo {
 	}
 	flat := d.flatten(*e)
 	return BucketUsageInfo{
-		Size:                 uint64(flat.Size),
-		ObjectsCount:         flat.Objects,
-		ObjectSizesHistogram: flat.ObjSizes.toMap(),
+		Size:                   uint64(flat.Size),
+		ObjectsCount:           flat.Objects,
+		ReplicationPendingSize: flat.ReplicationPendingSize,
+		ReplicatedSize:         flat.ReplicatedSize,
+		ReplicationFailedSize:  flat.ReplicationFailedSize,
+		ReplicaSize:            flat.ReplicaSize,
+		ObjectSizesHistogram:   flat.ObjSizes.toMap(),
 	}
 }
 
@@ -429,7 +474,7 @@ func (d *dataUsageCache) merge(other dataUsageCache) {
 }
 
 type objectIO interface {
-	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (reader *GetObjectReader, err error)
 	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 }
 
@@ -437,28 +482,35 @@ type objectIO interface {
 // Only backend errors are returned as errors.
 // If the object is not found or unable to deserialize d is cleared and nil error is returned.
 func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
-	var buf bytes.Buffer
-	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
+	r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
-		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) && !errors.Is(err, InsufficientReadQuorum{}) {
+		switch err.(type) {
+		case ObjectNotFound:
+		case BucketNotFound:
+		case InsufficientReadQuorum:
+		default:
 			return toObjectErr(err, dataUsageBucket, name)
 		}
 		*d = dataUsageCache{}
 		return nil
 	}
-	err = d.deserialize(&buf)
-	if err != nil {
+	defer r.Close()
+	if err := d.deserialize(r); err != nil {
 		*d = dataUsageCache{}
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, err.Error())
 	}
 	return nil
 }
 
 // save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
 func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
-	b := d.serialize()
-	size := int64(len(b))
-	r, err := hash.NewReader(bytes.NewReader(b), size, "", "", size, false)
+	pr, pw := io.Pipe()
+	go func() {
+		pw.CloseWithError(d.serializeTo(pw))
+	}()
+	defer pr.Close()
+
+	r, err := hash.NewReader(pr, -1, "", "", -1, false)
 	if err != nil {
 		return err
 	}
@@ -477,35 +529,40 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // dataUsageCacheVer indicates the cache version.
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
-const dataUsageCacheVer = 2
+const (
+	dataUsageCacheVerV3 = 3
+	dataUsageCacheVerV2 = 2
+	dataUsageCacheVerV1 = 1
+)
 
 // serialize the contents of the cache.
-func (d *dataUsageCache) serialize() []byte {
-	// Prepend version and compress.
-	dst := make([]byte, 0, d.Msgsize()+1)
-	dst = append(dst, dataUsageCacheVer)
-	buf := bytes.NewBuffer(dst)
-	enc, err := zstd.NewWriter(buf,
+func (d *dataUsageCache) serializeTo(dst io.Writer) error {
+	// Add version and compress.
+	_, err := dst.Write([]byte{dataUsageCacheVerV3})
+	if err != nil {
+		return err
+	}
+	enc, err := zstd.NewWriter(dst,
 		zstd.WithEncoderLevel(zstd.SpeedFastest),
 		zstd.WithWindowSize(1<<20),
 		zstd.WithEncoderConcurrency(2))
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return nil
+		return err
 	}
 	mEnc := msgp.NewWriter(enc)
 	err = d.EncodeMsg(mEnc)
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return nil
+		return err
+	}
+	err = mEnc.Flush()
+	if err != nil {
+		return err
 	}
-	mEnc.Flush()
 	err = enc.Close()
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return nil
+		return err
 	}
-	return buf.Bytes()
+	return nil
 }
 
 // deserialize the supplied byte slice into the cache.
@@ -516,21 +573,43 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		return io.ErrUnexpectedEOF
 	}
 	switch b[0] {
-	case 1:
+	case dataUsageCacheVerV1:
 		return errors.New("cache version deprecated (will autoupdate)")
-	case dataUsageCacheVer:
-	default:
-		return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
-	}
+	case dataUsageCacheVerV2:
+		// Zstd compressed.
+		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
+		if err != nil {
+			return err
+		}
+		defer dec.Close()
 
-	// Zstd compressed.
-	dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
-	if err != nil {
-		return err
-	}
-	defer dec.Close()
+		dold := &dataUsageCacheV2{}
+		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
+			return err
+		}
+		d.Info = dold.Info
+		d.Disks = dold.Disks
+		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
+		for k, v := range dold.Cache {
+			d.Cache[k] = dataUsageEntry{
+				Size:     v.Size,
+				Objects:  v.Objects,
+				ObjSizes: v.ObjSizes,
+				Children: v.Children,
+			}
+		}
+		return nil
+	case dataUsageCacheVerV3:
+		// Zstd compressed.
+		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
+		if err != nil {
+			return err
+		}
+		defer dec.Close()
 
-	return d.DecodeMsg(msgp.NewReader(dec))
+		return d.DecodeMsg(msgp.NewReader(dec))
+	}
+	return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
 }
 
 // Trim this from start+end of hashes.

cmd/data-usage-cache_gen.go

@@ -30,35 +30,54 @@ func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Cache":
+		case "Disks":
 			var zb0002 uint32
-			zb0002, err = dc.ReadMapHeader()
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Disks")
+				return
+			}
+			if cap(z.Disks) >= int(zb0002) {
+				z.Disks = (z.Disks)[:zb0002]
+			} else {
+				z.Disks = make([]string, zb0002)
+			}
+			for za0001 := range z.Disks {
+				z.Disks[za0001], err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Disks", za0001)
+					return
+				}
+			}
+		case "Cache":
+			var zb0003 uint32
+			zb0003, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntry, zb0002)
+				z.Cache = make(map[string]dataUsageEntry, zb0003)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0002 > 0 {
-				zb0002--
-				var za0001 string
-				var za0002 dataUsageEntry
-				za0001, err = dc.ReadString()
+			for zb0003 > 0 {
+				zb0003--
+				var za0002 string
+				var za0003 dataUsageEntry
+				za0002, err = dc.ReadString()
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				err = za0002.DecodeMsg(dc)
+				err = za0003.DecodeMsg(dc)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0001)
+					err = msgp.WrapError(err, "Cache", za0002)
 					return
 				}
-				z.Cache[za0001] = za0002
+				z.Cache[za0002] = za0003
 			}
 		default:
 			err = dc.Skip()
@@ -73,9 +92,9 @@ func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
+	// map header, size 3
 	// write "Info"
-	err = en.Append(0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	err = en.Append(0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
 	if err != nil {
 		return
 	}
@@ -84,6 +103,23 @@ func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Info")
 		return
 	}
+	// write "Disks"
+	err = en.Append(0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Disks)))
+	if err != nil {
+		err = msgp.WrapError(err, "Disks")
+		return
+	}
+	for za0001 := range z.Disks {
+		err = en.WriteString(z.Disks[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "Disks", za0001)
+			return
+		}
+	}
 	// write "Cache"
 	err = en.Append(0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
 	if err != nil {
@@ -94,15 +130,15 @@ func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Cache")
 		return
 	}
-	for za0001, za0002 := range z.Cache {
-		err = en.WriteString(za0001)
+	for za0002, za0003 := range z.Cache {
+		err = en.WriteString(za0002)
 		if err != nil {
 			err = msgp.WrapError(err, "Cache")
 			return
 		}
-		err = za0002.EncodeMsg(en)
+		err = za0003.EncodeMsg(en)
 		if err != nil {
-			err = msgp.WrapError(err, "Cache", za0001)
+			err = msgp.WrapError(err, "Cache", za0002)
 			return
 		}
 	}
@@ -112,22 +148,28 @@ func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageCache) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
+	// map header, size 3
 	// string "Info"
-	o = append(o, 0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	o = append(o, 0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
 	o, err = z.Info.MarshalMsg(o)
 	if err != nil {
 		err = msgp.WrapError(err, "Info")
 		return
 	}
+	// string "Disks"
+	o = append(o, 0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Disks)))
+	for za0001 := range z.Disks {
+		o = msgp.AppendString(o, z.Disks[za0001])
+	}
 	// string "Cache"
 	o = append(o, 0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
 	o = msgp.AppendMapHeader(o, uint32(len(z.Cache)))
-	for za0001, za0002 := range z.Cache {
-		o = msgp.AppendString(o, za0001)
-		o, err = za0002.MarshalMsg(o)
+	for za0002, za0003 := range z.Cache {
+		o = msgp.AppendString(o, za0002)
+		o, err = za0003.MarshalMsg(o)
 		if err != nil {
-			err = msgp.WrapError(err, "Cache", za0001)
+			err = msgp.WrapError(err, "Cache", za0002)
 			return
 		}
 	}
@@ -158,35 +200,54 @@ func (z *dataUsageCache) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Cache":
+		case "Disks":
 			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Disks")
+				return
+			}
+			if cap(z.Disks) >= int(zb0002) {
+				z.Disks = (z.Disks)[:zb0002]
+			} else {
+				z.Disks = make([]string, zb0002)
+			}
+			for za0001 := range z.Disks {
+				z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Disks", za0001)
+					return
+				}
+			}
+		case "Cache":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntry, zb0002)
+				z.Cache = make(map[string]dataUsageEntry, zb0003)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0002 > 0 {
-				var za0001 string
-				var za0002 dataUsageEntry
-				zb0002--
-				za0001, bts, err = msgp.ReadStringBytes(bts)
+			for zb0003 > 0 {
+				var za0002 string
+				var za0003 dataUsageEntry
+				zb0003--
+				za0002, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				bts, err = za0002.UnmarshalMsg(bts)
+				bts, err = za0003.UnmarshalMsg(bts)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0001)
+					err = msgp.WrapError(err, "Cache", za0002)
 					return
 				}
-				z.Cache[za0001] = za0002
+				z.Cache[za0002] = za0003
 			}
 		default:
 			bts, err = msgp.Skip(bts)
@@ -202,11 +263,15 @@ func (z *dataUsageCache) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCache) Msgsize() (s int) {
-	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize
+	for za0001 := range z.Disks {
+		s += msgp.StringPrefixSize + len(z.Disks[za0001])
+	}
+	s += 6 + msgp.MapHeaderSize
 	if z.Cache != nil {
-		for za0001, za0002 := range z.Cache {
-			_ = za0002
-			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
+		for za0002, za0003 := range z.Cache {
+			_ = za0003
+			s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize()
 		}
 	}
 	return
@@ -419,8 +484,500 @@ func (z *dataUsageCacheInfo) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageCacheV2) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			err = z.Info.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Disks":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Disks")
+				return
+			}
+			if cap(z.Disks) >= int(zb0002) {
+				z.Disks = (z.Disks)[:zb0002]
+			} else {
+				z.Disks = make([]string, zb0002)
+			}
+			for za0001 := range z.Disks {
+				z.Disks[za0001], err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Disks", za0001)
+					return
+				}
+			}
+		case "Cache":
+			var zb0003 uint32
+			zb0003, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntryV2, zb0003)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0003 > 0 {
+				zb0003--
+				var za0002 string
+				var za0003 dataUsageEntryV2
+				za0002, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				err = za0003.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0002)
+					return
+				}
+				z.Cache[za0002] = za0003
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *dataUsageCacheV2) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 3
+	// write "Info"
+	err = en.Append(0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	if err != nil {
+		return
+	}
+	err = z.Info.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Info")
+		return
+	}
+	// write "Disks"
+	err = en.Append(0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Disks)))
+	if err != nil {
+		err = msgp.WrapError(err, "Disks")
+		return
+	}
+	for za0001 := range z.Disks {
+		err = en.WriteString(z.Disks[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "Disks", za0001)
+			return
+		}
+	}
+	// write "Cache"
+	err = en.Append(0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.Cache)))
+	if err != nil {
+		err = msgp.WrapError(err, "Cache")
+		return
+	}
+	for za0002, za0003 := range z.Cache {
+		err = en.WriteString(za0002)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache")
+			return
+		}
+		err = za0003.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache", za0002)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *dataUsageCacheV2) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 3
+	// string "Info"
+	o = append(o, 0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	o, err = z.Info.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Info")
+		return
+	}
+	// string "Disks"
+	o = append(o, 0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Disks)))
+	for za0001 := range z.Disks {
+		o = msgp.AppendString(o, z.Disks[za0001])
+	}
+	// string "Cache"
+	o = append(o, 0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Cache)))
+	for za0002, za0003 := range z.Cache {
+		o = msgp.AppendString(o, za0002)
+		o, err = za0003.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache", za0002)
+			return
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageCacheV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			bts, err = z.Info.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Disks":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Disks")
+				return
+			}
+			if cap(z.Disks) >= int(zb0002) {
+				z.Disks = (z.Disks)[:zb0002]
+			} else {
+				z.Disks = make([]string, zb0002)
+			}
+			for za0001 := range z.Disks {
+				z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Disks", za0001)
+					return
+				}
+			}
+		case "Cache":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntryV2, zb0003)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0003 > 0 {
+				var za0002 string
+				var za0003 dataUsageEntryV2
+				zb0003--
+				za0002, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				bts, err = za0003.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0002)
+					return
+				}
+				z.Cache[za0002] = za0003
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageCacheV2) Msgsize() (s int) {
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize
+	for za0001 := range z.Disks {
+		s += msgp.StringPrefixSize + len(z.Disks[za0001])
+	}
+	s += 6 + msgp.MapHeaderSize
+	if z.Cache != nil {
+		for za0002, za0003 := range z.Cache {
+			_ = za0003
+			s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize()
+		}
+	}
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 8 {
+		err = msgp.ArrayError{Wanted: 8, Got: zb0001}
+		return
+	}
+	z.Size, err = dc.ReadInt64()
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.ReplicatedSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	z.ReplicationPendingSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationPendingSize")
+		return
+	}
+	z.ReplicationFailedSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationFailedSize")
+		return
+	}
+	z.ReplicaSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
+	z.Objects, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.DecodeMsg(dc)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
+	// array header, size 8
+	err = en.Append(0x98)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.Size)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	err = en.WriteUint64(z.ReplicatedSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	err = en.WriteUint64(z.ReplicationPendingSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationPendingSize")
+		return
+	}
+	err = en.WriteUint64(z.ReplicationFailedSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationFailedSize")
+		return
+	}
+	err = en.WriteUint64(z.ReplicaSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
+	err = en.WriteUint64(z.Objects)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		err = en.WriteUint64(z.ObjSizes[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// array header, size 8
+	o = append(o, 0x98)
+	o = msgp.AppendInt64(o, z.Size)
+	o = msgp.AppendUint64(o, z.ReplicatedSize)
+	o = msgp.AppendUint64(o, z.ReplicationPendingSize)
+	o = msgp.AppendUint64(o, z.ReplicationFailedSize)
+	o = msgp.AppendUint64(o, z.ReplicaSize)
+	o = msgp.AppendUint64(o, z.Objects)
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
+	for za0001 := range z.ObjSizes {
+		o = msgp.AppendUint64(o, z.ObjSizes[za0001])
+	}
+	o, err = z.Children.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 8 {
+		err = msgp.ArrayError{Wanted: 8, Got: zb0001}
+		return
+	}
+	z.Size, bts, err = msgp.ReadInt64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.ReplicatedSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	z.ReplicationPendingSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationPendingSize")
+		return
+	}
+	z.ReplicationFailedSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationFailedSize")
+		return
+	}
+	z.ReplicaSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
+	z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	bts, err = z.Children.UnmarshalMsg(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageEntry) Msgsize() (s int) {
+	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageEntryV2) DecodeMsg(dc *msgp.Reader) (err error) {
 	var zb0001 uint32
 	zb0001, err = dc.ReadArrayHeader()
 	if err != nil {
@@ -467,7 +1024,7 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 }
 
 // EncodeMsg implements msgp.Encodable
-func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
+func (z *dataUsageEntryV2) EncodeMsg(en *msgp.Writer) (err error) {
 	// array header, size 4
 	err = en.Append(0x94)
 	if err != nil {
@@ -504,7 +1061,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 }
 
 // MarshalMsg implements msgp.Marshaler
-func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
+func (z *dataUsageEntryV2) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// array header, size 4
 	o = append(o, 0x94)
@@ -523,7 +1080,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
-func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+func (z *dataUsageEntryV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var zb0001 uint32
 	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
 	if err != nil {
@@ -571,7 +1128,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *dataUsageEntry) Msgsize() (s int) {
+func (z *dataUsageEntryV2) Msgsize() (s int) {
 	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
 	return
 }