handle trimming '/' if present in the object names (#11765)

- MultipleDeletes should handle '/' prefix for objectnames
- Trimming the slash alone is enough for ListObjects()
  prefix and markers

fixes #11769

.dockerignore

@@ -1,2 +1,9 @@
 .git
 .github
+docs
+default.etcd
+browser
+*.gz
+*.tar.gz
+*.bzip2
+*.zip

.github/workflows/go.yml

@@ -11,8 +11,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.14.x, 1.15.x]
-        os: [ubuntu-latest, windows-latest]
+        go-version: [1.16.x]
+        os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
@@ -21,6 +21,14 @@ jobs:
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
+      - name: Build on ${{ matrix.os }}
+        if: matrix.os == 'macos-latest'
+        env:
+          CGO_ENABLED: 0
+          GO111MODULE: on
+        run: |
+          make
+          make test-race
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'windows-latest'
         env:
@@ -42,8 +50,6 @@ jobs:
           curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-${nancy_version}-linux-amd64 && chmod +x nancy
           go list -m all | ./nancy sleuth
           make
-          diff -au <(gofmt -s -d cmd) <(printf "")
-          diff -au <(gofmt -s -d pkg) <(printf "")
           make test-race
           make crosscompile
           make verify

.golangci.yml

@@ -17,6 +17,8 @@ linters:
     - gosimple
     - deadcode
     - structcheck
+    - gomodguard
+    - gofmt
 
 issues:
   exclude-use-default: false

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15-alpine as builder
+FROM golang:1.16-alpine as builder
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 

Dockerfile.cicd

@@ -1,4 +1,4 @@
-FROM golang:1.15-alpine as builder
+FROM golang:1.16-alpine as builder
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 

Dockerfile.dev

@@ -6,8 +6,6 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 COPY minio /usr/bin/
-COPY CREDITS /licenses/CREDITS
-COPY LICENSE /licenses/LICENSE
 
 ENV MINIO_UPDATE=off \
     MINIO_ACCESS_KEY_FILE=access_key \
@@ -17,10 +15,9 @@ ENV MINIO_UPDATE=off \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
     MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
-RUN  \
-    microdnf update --nodocs && \
-    microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
-    microdnf clean all && \
+RUN microdnf update --nodocs
+RUN microdnf install curl ca-certificates shadow-utils util-linux --nodocs
+RUN microdnf clean all && \
     chmod +x /usr/bin/minio  && \
     chmod +x /usr/bin/docker-entrypoint.sh
 

Dockerfile.release

@@ -2,11 +2,13 @@ FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
 
 ARG TARGETARCH
 
+ARG RELEASE
+
 LABEL name="MinIO" \
       vendor="MinIO Inc <dev@min.io>" \
       maintainer="MinIO Inc <dev@min.io>" \
-      version="RELEASE.2021-01-08T21-18-21Z" \
-      release="RELEASE.2021-01-08T21-18-21Z" \
+      version="${RELEASE}" \
+      release="${RELEASE}" \
       summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
       description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
 
@@ -28,9 +30,9 @@ RUN \
      microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
      rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
      microdnf install minisign --nodocs && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio.minisig -o /usr/bin/minio.minisig && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /usr/bin/minio && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /usr/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /usr/bin/minio.minisig && \
      microdnf clean all && \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \

Makefile

@@ -17,32 +17,22 @@ checks:
 getdeps:
 	@mkdir -p ${GOPATH}/bin
 	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
-	@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && go get github.com/quasilyte/go-ruleguard/cmd/ruleguard@v0.2.1)
 	@which msgp 1>/dev/null || (echo "Installing msgp" && go get github.com/tinylib/msgp@v1.1.3)
 	@which stringer 1>/dev/null || (echo "Installing stringer" && go get golang.org/x/tools/cmd/stringer)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
-verifiers: getdeps fmt lint ruleguard check-gen
+verifiers: getdeps lint check-gen
 
 check-gen:
 	@go generate ./... >/dev/null
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
 
-fmt:
-	@echo "Running $@ check"
-	@GO111MODULE=on gofmt -d cmd/
-	@GO111MODULE=on gofmt -d pkg/
-
 lint:
 	@echo "Running $@ check"
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
-	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
-
-ruleguard:
-	@echo "Running $@ check"
-	@${GOPATH}/bin/ruleguard -rules ruleguard.rules.go github.com/minio/minio/...
+	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
@@ -71,13 +61,18 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
-hotfix: LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
-    sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#'))
-hotfix: install
+hotfix-vars:
+	$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
+    sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#')))
+	$(eval TAG := "minio/minio:$(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD)")
+hotfix: hotfix-vars install
 
-docker: checks
+docker-hotfix: hotfix checks
+	@echo "Building minio docker image '$(TAG)'"
+	@docker build -t $(TAG) . -f Dockerfile.dev
+
+docker: build checks
 	@echo "Building minio docker image '$(TAG)'"
-	@GOOS=linux GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@docker build -t $(TAG) . -f Dockerfile.dev
 
 # Builds minio and installs it to $GOPATH/bin.

README.md

@@ -5,47 +5,32 @@
 
 MinIO is a High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
 
-This README provides quickstart instructions on running MinIO on baremetal hardware, including Docker-based installations. For Kubernetes environments, 
-use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md). 
-
-**Table of Contents**
-
-- [Docker Installation](#docker-installation)
-- [macOS Installation](#macos)
-- [GNU/Linux Installation](#gnulinux)
-- [Windows Installation](#microsoft-windows)
-- [FreeBSD Installation](#freebsd)
-- [Source Installation](#install-from-source)
-- [Deployment Recommendations](#deployment-recommendations)
-- [Test Connectivity to MinIO](#test-minio-connectivity)
-- [Upgrade MinIO](#upgrading-minio)
-- [Explore Further](#explore-further)
-- [Contribute to MinIO](#contribute-to-minio-project)
-- [License](#license)
+This README provides quickstart instructions on running MinIO on baremetal hardware, including Docker-based installations. For Kubernetes environments,
+use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md).
 
 # Docker Installation
 
-Use the following commands to run a standalone MinIO server on a Docker container. 
+Use the following commands to run a standalone MinIO server on a Docker container.
 
-Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
-require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
-with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
 for more complete documentation.
 
 ## Stable
 
-Run the following command to run the latest stable image of MinIO on a Docker container using an ephemeral data volume: 
+Run the following command to run the latest stable image of MinIO on a Docker container using an ephemeral data volume:
 
 ```sh
 docker run -p 9000:9000 minio/minio server /data
 ```
 
-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
-web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
 root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
-[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
 see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
 
 
@@ -60,12 +45,12 @@ Run the following command to run the bleeding-edge image of MinIO on a Docker co
 docker run -p 9000:9000 minio/minio:edge server /data
 ```
 
-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
-web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
 root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
-[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
 see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
 
 
@@ -76,9 +61,9 @@ see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view
 
 Use the following commands to run a standalone MinIO server on macOS.
 
-Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
-require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
-with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
 for more complete documentation.
 
 ## Homebrew (recommended)
@@ -97,12 +82,12 @@ brew uninstall minio
 brew install minio/stable/minio
 ```
 
-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
-web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
 root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
-[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
 see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
 
 ## Binary Download
@@ -115,12 +100,12 @@ chmod +x minio
 ./minio server /data
 ```
 
-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
-web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
 root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
-[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
 see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
 
 
@@ -145,18 +130,18 @@ The following table lists supported architectures. Replace the `wget` URL with t
 | 64-bit PowerPC LE (ppc64le)    | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
 | IBM Z-Series (S390X)           | https://dl.min.io/server/minio/release/linux-s390x/minio   |
 
-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
-web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
 root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
-[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
 see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
 
 
-> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
-require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
-with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
 for more complete documentation.
 
 # Microsoft Windows
@@ -173,17 +158,17 @@ Use the following command to run a standalone MinIO server on the Windows host.
 minio.exe server D:\
 ```
 
-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
-web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
 root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
-[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
 see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
 
-> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
-require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
-with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
 for more complete documentation.
 
 # FreeBSD
@@ -199,27 +184,27 @@ service minio start
 
 # Install from Source
 
-Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.16](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio
 ```
 
-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
-web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
 root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
-[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
 see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
 
 
-> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
-require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
-with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
-for more complete documentation. 
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
+for more complete documentation.
 
-MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments. 
+MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments.
 
 # Deployment Recommendations
 

README_zh_CN.md

@@ -78,7 +78,7 @@ minio.exe server D:\Photos
 
 ## FreeBSD
 ### Port
-使用 [pkg](https://github.com/freebsd/pkg)进行安装，, MinIO官方并没有提供FreeBSD二进制文件， 它由FreeBSD上游维护，点击 [这里](https://www.freshports.org/www/minio)查看。
+使用 [pkg](https://github.com/freebsd/pkg)进行安装，MinIO官方并没有提供FreeBSD二进制文件， 它由FreeBSD上游维护，点击 [这里](https://www.freshports.org/www/minio)查看。
 
 ```sh
 pkg install minio
@@ -89,7 +89,7 @@ service minio start
 
 ## 使用源码安装
 
-采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install)。最低需要Golang版本为 [go1.14](https://golang.org/dl/#stable)
+采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install)。最低需要Golang版本为 [go1.16](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio
@@ -179,7 +179,7 @@ mc admin update <minio alias, e.g., myminio>
 - 对于联盟部署模式，应分别针对每个群集运行`mc admin update`。 在成功更新所有群集之前，不要将`mc`更新为任何新版本。
 - 如果将`kes`用作MinIO的KMS，只需替换二进制文件并重新启动`kes`，可以在 [这里](https://github.com/minio/kes/wiki) 找到有关`kes`的更多信息。
 - 如果将Vault作为MinIO的KMS，请确保已遵循如下Vault升级过程的概述：https://www.vaultproject.io/docs/upgrading/index.html
-- 如果将MindIO与etcd配合使用, 请确保已遵循如下etcd升级过程的概述: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
+- 如果将MinIO与etcd配合使用, 请确保已遵循如下etcd升级过程的概述: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
 
 ## 了解更多
 - [MinIO纠删码入门](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
@@ -190,7 +190,7 @@ mc admin update <minio alias, e.g., myminio>
 - [MinIO文档](https://docs.min.io)
 
 ## 如何参与到MinIO项目
-请参考 [贡献者指南](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)。欢迎各位中国程序员加到MinIO项目中。
+请参考 [贡献者指南](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)。欢迎各位中国程序员加入到MinIO项目中。
 
 ## 授权许可
 MinIO的使用受 Apache 2.0 License 约束，你可以在 [LICENSE](./LICENSE) 查看许可。

browser/.gitignore

@@ -17,4 +17,3 @@ release
 *.syso
 coverage.txt
 node_modules
-production

browser/README.md

@@ -17,24 +17,13 @@ nvm install stable
 npm install
 ```
 
-### Install `go-bindata` and `go-bindata-assetfs`
-
-If you do not have a working Golang environment, please follow [Install Golang](https://golang.org/doc/install)
-
-```sh
-go get github.com/go-bindata/go-bindata/go-bindata
-go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs
-```
-
 ## Generating Assets
 
-### Generate ui-assets.go
-
 ```sh
 npm run release
 ```
 
-This generates ui-assets.go in the current directory. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
+This generates `production` in the current directory. 
 
 
 ## Run MinIO Browser with live reload

browser/app/js/browser/__tests__/ChangePasswordModal.test.js

@@ -24,9 +24,6 @@ jest.mock("jwt-decode")
 jwtDecode.mockImplementation(() => ({ sub: "minio" }))
 
 jest.mock("../../web", () => ({
-  GenerateAuth: jest.fn(() => {
-    return Promise.resolve({ accessKey: "gen1", secretKey: "gen2" })
-  }),
   SetAuth: jest.fn(
     ({ currentAccessKey, currentSecretKey, newAccessKey, newSecretKey }) => {
       if (

browser/app/js/objects/ShareObjectModal.js

@@ -26,6 +26,7 @@ import {
   SHARE_OBJECT_EXPIRY_HOURS,
   SHARE_OBJECT_EXPIRY_MINUTES
 } from "../constants"
+import QRCode from "react-qr-code";
 
 export class ShareObjectModal extends React.Component {
   constructor(props) {
@@ -89,6 +90,7 @@ export class ShareObjectModal extends React.Component {
         <ModalHeader>Share Object</ModalHeader>
         <ModalBody>
           <div className="input-group copy-text">
+            <QRCode value={url} size={128}/>
             <label>Shareable Link</label>
             <input
               type="text"

browser/app/less/inc/misc.less

@@ -75,6 +75,11 @@
 			border-color: darken(@input-border, 5%);
 		}
 	}
+
+	svg {
+	  display: block;
+		margin: 0 auto 5px;
+	}
 }
 
 /*--------------------------
@@ -150,4 +155,4 @@
   100% {
     transform: rotate(360deg);
   }
-}
+}

browser/assets.go

@@ -0,0 +1,11 @@
+package browser
+
+import "embed"
+
+//go:embed production/*
+var fs embed.FS
+
+// GetStaticAssets returns assets
+func GetStaticAssets() embed.FS {
+	return fs
+}

browser/build.js

@@ -14,19 +14,11 @@
  * limitations under the License.
  */
 
-var moment = require('moment')
 var async = require('async')
 var exec = require('child_process').exec
 var fs = require('fs')
 
 var isProduction = process.env.NODE_ENV == 'production' ? true : false
-var assetsFileName = ''
-var commitId = ''
-var date = moment.utc()
-var version = date.format('YYYY-MM-DDTHH:mm:ss') + 'Z'
-var releaseTag = date.format('YYYY-MM-DDTHH-mm-ss') + 'Z'
-var buildType = 'DEVELOPMENT'
-if (process.env.MINIO_UI_BUILD) buildType = process.env.MINIO_UI_BUILD
 
 rmDir = function(dirPath) {
   try { var files = fs.readdirSync(dirPath); }
@@ -53,74 +45,6 @@ async.waterfall([
       console.log('Running', cmd)
       exec(cmd, cb)
     },
-    function(stdout, stderr, cb) {
-      if (isProduction) {
-        fs.renameSync('production/index_bundle.js',
-                      'production/index_bundle-' + releaseTag + '.js')
-      } else {
-        fs.renameSync('dev/index_bundle.js',
-                      'dev/index_bundle-' + releaseTag + '.js')
-      }
-      var cmd = 'git log --format="%H" -n1'
-      console.log('Running', cmd)
-      exec(cmd, cb)
-    },
-    function(stdout, stderr, cb) {
-      if (!stdout) throw new Error('commitId is empty')
-      commitId = stdout.replace('\n', '')
-      if (commitId.length !== 40) throw new Error('commitId invalid : ' + commitId)
-      assetsFileName = 'ui-assets.go';
-      var cmd = 'go-bindata-assetfs -o bindata_assetfs.go -pkg browser -nocompress=true production/...'
-      if (!isProduction) {
-        cmd = 'go-bindata-assetfs -o bindata_assetfs.go -pkg browser -nocompress=true dev/...'
-      }
-      console.log('Running', cmd)
-      exec(cmd, cb)
-    },
-    function(stdout, stderr, cb) {
-      var cmd = 'gofmt -s -w -l bindata_assetfs.go'
-      console.log('Running', cmd)
-      exec(cmd, cb)
-    },
-    function(stdout, stderr, cb) {
-      fs.renameSync('bindata_assetfs.go', assetsFileName)
-      fs.appendFileSync(assetsFileName, '\n')
-      fs.appendFileSync(assetsFileName, 'var UIReleaseTag = "' + buildType + '.' +
-                        releaseTag + '"\n')
-      fs.appendFileSync(assetsFileName, 'var UICommitID = "' + commitId + '"\n')
-      fs.appendFileSync(assetsFileName, 'var UIVersion = "' + version + '"')
-      fs.appendFileSync(assetsFileName, '\n')
-      var contents;
-      if (isProduction) {
-        contents = fs.readFileSync(assetsFileName, 'utf8')
-                     .replace(/_productionIndexHtml/g, '_productionIndexHTML')
-                     .replace(/productionIndexHtmlBytes/g, 'productionIndexHTMLBytes')
-                     .replace(/productionIndexHtml/g, 'productionIndexHTML')
-                     .replace(/_productionIndex_bundleJs/g, '_productionIndexBundleJs')
-                     .replace(/productionIndex_bundleJsBytes/g, 'productionIndexBundleJsBytes')
-                     .replace(/productionIndex_bundleJs/g, 'productionIndexBundleJs')
-                     .replace(/_productionJqueryUiMinJs/g, '_productionJqueryUIMinJs')
-                     .replace(/productionJqueryUiMinJsBytes/g, 'productionJqueryUIMinJsBytes')
-                     .replace(/productionJqueryUiMinJs/g, 'productionJqueryUIMinJs');
-      } else {
-        contents = fs.readFileSync(assetsFileName, 'utf8')
-                     .replace(/_devIndexHtml/g, '_devIndexHTML')
-                     .replace(/devIndexHtmlBytes/g, 'devIndexHTMLBytes')
-                     .replace(/devIndexHtml/g, 'devIndexHTML')
-                     .replace(/_devIndex_bundleJs/g, '_devIndexBundleJs')
-                     .replace(/devIndex_bundleJsBytes/g, 'devIndexBundleJsBytes')
-                     .replace(/devIndex_bundleJs/g, 'devIndexBundleJs')
-                     .replace(/_devJqueryUiMinJs/g, '_devJqueryUIMinJs')
-                     .replace(/devJqueryUiMinJsBytes/g, 'devJqueryUIMinJsBytes')
-                     .replace(/devJqueryUiMinJs/g, 'devJqueryUIMinJs');
-      }
-      contents = contents.replace(/MINIO_UI_VERSION/g, version)
-      contents = contents.replace(/index_bundle.js/g, 'index_bundle-' + releaseTag + '.js')
-
-      fs.writeFileSync(assetsFileName, contents, 'utf8')
-      console.log('UI assets file :', assetsFileName)
-      cb()
-    }
   ], function(err) {
     if (err) return console.log(err)
   })

browser/package.json

@@ -6,7 +6,7 @@
     "test": "jest",
     "dev": "NODE_ENV=dev webpack-dev-server --devtool cheap-module-eval-source-map --progress --colors --hot --content-base dev",
     "build": "NODE_ENV=dev node build.js",
-    "release": "NODE_ENV=production MINIO_UI_BUILD=RELEASE node build.js",
+    "release": "NODE_ENV=production node build.js",
     "format": "esformatter -i 'app/**/*.js'"
   },
   "jest": {
@@ -84,6 +84,7 @@
     "react-dropzone": "^11.0.1",
     "react-infinite-scroller": "^1.2.4",
     "react-onclickout": "^2.0.8",
+    "react-qr-code": "^1.1.1",
     "react-redux": "^5.1.2",
     "react-router-dom": "^5.2.0",
     "redux": "^4.0.5",

browser/production/index.html

@@ -0,0 +1,59 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1">
+        <title>MinIO Browser</title>
+
+        <link rel="icon" type="image/png" sizes="32x32" href="/minio/favicon-32x32.png">
+        <link rel="icon" type="image/png" sizes="96x96" href="/minio/favicon-96x96.png">
+        <link rel="icon" type="image/png" sizes="16x16" href="/minio/favicon-16x16.png">
+
+        <link rel="stylesheet" href="/minio/loader.css" type="text/css">
+    </head>
+
+    <body>
+        <div class="page-load">
+            <div class="pl-inner">
+                <img src="/minio/logo.svg" alt="">
+            </div>
+        </div>
+        <div id="root"></div>
+
+        <!--[if lt IE 11]>
+            <div class="ie-warning">
+                <div class="iw-inner">
+                    <i class="iwi-icon fas fa-exclamation-triangle"></i>
+
+                    You are using Internet Explorer version 12.0 or lower. Due to security issues and lack of support for Web Standards it is highly recommended that you upgrade to a modern browser
+
+                    <ul>
+                        <li>
+                            <a href="http://www.google.com/chrome/">
+                                <img src="chrome.png" alt="">
+                                <div>Chrome</div>
+                            </a>
+                        </li>
+                        <li>
+                            <a href="https://www.mozilla.org/en-US/firefox/new/">
+                                <img src="firefox.png" alt="">
+                                <div>Firefox</div>
+                            </a>
+                        </li>
+                        <li>
+                            <a href="https://www.apple.com/safari/">
+                                <img src="safari.png" alt="">
+                                <div>Safari</div>
+                            </a>
+                        </li>
+                    </ul>
+
+                    <div class="iwi-skip">Skip & Continue</div>
+                </div>
+            </div>
+        <![endif]-->
+
+        <script>currentUiVersion = 'MINIO_UI_VERSION'</script>
+        <script src="/minio/index_bundle.js"></script>
+    </body>
+</html>

browser/production/loader.css

@@ -0,0 +1,98 @@
+.page-load {
+    position: fixed;
+    width: 100%;
+    height: 100%;
+    top: 0;
+    left: 0;
+    background: #002a37;
+    z-index: 100;
+    transition: opacity 200ms;
+    -webkit-transition: opacity 200ms;
+}
+
+.pl-0{
+    opacity: 0;
+}
+
+.pl-1 {
+    display: none;
+}
+
+.pl-inner {
+    position: absolute;
+    width: 100px;
+    height: 100px;
+    left: 50%;
+    margin-left: -50px;
+    top: 50%;
+    margin-top: -50px;
+    text-align: center;
+    -webkit-animation: fade-in 500ms;
+    animation: fade-in 500ms;
+    -webkit-animation-fill-mode: both;
+    animation-fill-mode: both;
+    animation-delay: 350ms;
+    -webkit-animation-delay: 350ms;
+    -webkit-backface-visibility: visible;
+    backface-visibility: visible;
+}
+
+.pl-inner:before {
+    content: '';
+    position: absolute;
+    width: 100%;
+    height: 100%;
+    left: 0;
+    top: 0;
+    display: block;
+    -webkit-animation: spin 1000ms infinite linear;
+    animation: spin 1000ms infinite linear;
+    border: 1px solid rgba(255, 255, 255, 0.2);;
+    border-left-color: #fff;
+    border-radius: 50%;
+}
+
+.pl-inner > img {
+    width: 30px;
+    margin-top: 21px;
+}
+
+@-webkit-keyframes fade-in {
+    0% {
+        opacity: 0;
+    }
+    100% {
+        opacity: 1;
+    }
+}
+
+@keyframes fade-in {
+    0% {
+        opacity: 0;
+    }
+    100% {
+        opacity: 1;
+    }
+}
+
+@-webkit-keyframes spin {
+    0% {
+        -webkit-transform: rotate(0deg);
+        transform: rotate(0deg);
+    }
+    100% {
+        -webkit-transform: rotate(360deg);
+        transform: rotate(360deg);
+    }
+}
+
+@keyframes spin {
+    0% {
+        -webkit-transform: rotate(0deg);
+        transform: rotate(0deg);
+    }
+    100% {
+        -webkit-transform: rotate(360deg);
+        transform: rotate(360deg);
+    }
+}

browser/production/logo.svg

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="93px" height="187px" viewBox="0 0 93 187" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 48.2 (47327) - http://www.bohemiancoding.com/sketch -->
+    <title>logo</title>
+    <desc>Created with Sketch.</desc>
+    <defs></defs>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="logo" transform="translate(0.187500, -0.683594)" fill="#FFFFFF" fill-rule="nonzero">
+            <path d="M91.49,46.551 C86.7827023,38.7699609 82.062696,30.9966172 77.33,23.231 C74.87,19.231 72.33,15.231 69.88,11.231 C69.57,10.731 69.18,10.291 68.88,9.831 C64.35,2.931 55.44,-1.679 46.73,2.701 C42.9729806,4.51194908 40.0995718,7.75449451 38.7536428,11.7020516 C37.4077139,15.6496086 37.701799,19.9721186 39.57,23.701 C41.08,26.641 43.57,29.121 45.91,31.581 C53.03,39.141 60.38,46.491 67.45,54.111 C72.4175495,59.4492221 74.4526451,66.8835066 72.8965704,74.0075359 C71.3404956,81.1315653 66.390952,87.0402215 59.65,89.821 C59.4938176,89.83842 59.3361824,89.83842 59.18,89.821 L59.18,54.591 C46.6388051,61.0478363 35.3944735,69.759905 26.01,80.291 C11.32,96.671 2.64,117.141 0.01,132.071 L23.96,119.821 C31.96,115.771 39.86,111.821 48.14,107.581 L48.14,175.921 L59.14,187.131 L59.14,101.831 C59.14,101.831 59.39,101.711 60.22,101.261 C63.5480598,99.6738911 66.7772674,97.8873078 69.89,95.911 C77.7130888,90.4306687 82.7479457,81.8029342 83.6709542,72.295947 C84.5939627,62.7889599 81.3127806,53.3538429 74.69,46.471 C66.49,37.891 58.24,29.351 50.05,20.761 C47.67,18.261 47.72,15.101 50.05,12.881 C52.38,10.661 55.56,10.881 57.96,13.331 L61.38,16.781 C64.1,19.681 66.79,22.611 69.53,25.481 C76.4547149,32.7389629 83.3947303,39.9823123 90.35,47.211 C90.7,47.571 91.12,47.871 91.5,48.211 L91.93,47.951 C91.8351945,47.4695902 91.6876376,47.0000911 91.49,46.551 Z M48.11,94.931 C47.9883217,95.5022568 47.6230065,95.9917791 47.11,96.271 C42.72,98.601 38.29,100.871 33.87,103.141 L17.76,111.401 C24.771203,96.7435071 35.1132853,83.9289138 47.96,73.981 C48.08,74.221 48.16,74.301 48.16,74.381 C48.15,81.231 48.17,88.081 48.11,94.931 Z" id="Shape"></path>
+        </g>
+    </g>
+</svg>

buildscripts/checkdeps.sh

@@ -21,7 +21,7 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GO_VERSION="1.13"
+    GO_VERSION="1.16"
     OSX_VERSION="10.8"
     KNAME=$(uname -s)
     ARCH=$(uname -m)

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64"
+    SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64"
 }
 
 function _build() {

buildscripts/race.sh

@@ -3,5 +3,5 @@
 set -e
 
 for d in $(go list ./... | grep -v browser); do
-    CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
+    CGO_ENABLED=1 go test -v -tags kqueue -race --timeout 100m "$d"
 done

cmd/acl-handlers.go

@@ -20,7 +20,6 @@ import (
 	"encoding/xml"
 	"io"
 	"net/http"
-	"net/url"
 
 	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -61,7 +60,7 @@ type accessControlPolicy struct {
 func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketACL")
 
-	defer logger.AuditLog(w, r, "PutBucketACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -125,7 +124,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketACL")
 
-	defer logger.AuditLog(w, r, "GetBucketACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -176,11 +175,11 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectACL")
 
-	defer logger.AuditLog(w, r, "PutObjectACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
-	object, err := url.PathUnescape(vars["object"])
+	object, err := unescapePath(vars["object"])
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -240,11 +239,11 @@ func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectACL")
 
-	defer logger.AuditLog(w, r, "GetObjectACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
-	object, err := url.PathUnescape(vars["object"])
+	object, err := unescapePath(vars["object"])
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/admin-bucket-handlers.go

@@ -41,7 +41,7 @@ const (
 func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketQuotaConfig")
 
-	defer logger.AuditLog(w, r, "PutBucketQuotaConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
 	if objectAPI == nil {
@@ -81,7 +81,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketQuotaConfig")
 
-	defer logger.AuditLog(w, r, "GetBucketQuotaConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
 	if objectAPI == nil {
@@ -116,7 +116,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetBucketTarget")
 
-	defer logger.AuditLog(w, r, "SetBucketTarget", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	update := r.URL.Query().Get("update") == "true"
@@ -156,7 +156,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-
 	sameTarget, _ := isLocalHost(target.URL().Hostname(), target.URL().Port(), globalMinioPort)
 	if sameTarget && bucket == target.TargetBucket {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL)
@@ -205,7 +204,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBucketTargets")
 
-	defer logger.AuditLog(w, r, "ListBucketTargets", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	arnType := vars["type"]
@@ -244,7 +243,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveBucketTarget")
 
-	defer logger.AuditLog(w, r, "RemoveBucketTarget", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	arn := vars["arn"]

cmd/admin-handlers-config-kv.go

@@ -62,7 +62,7 @@ func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *htt
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteConfigKV")
 
-	defer logger.AuditLog(w, r, "DeleteConfigKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -104,7 +104,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfigKV")
 
-	defer logger.AuditLog(w, r, "SetConfigKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -130,13 +130,14 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
 	dynamic, err := cfg.ReadConfig(bytes.NewReader(kvBytes))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -158,15 +159,14 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		saveConfig(GlobalContext, objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 	}
 
-	// Apply dynamic values.
-	if err := applyDynamicConfig(GlobalContext, cfg); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	globalNotificationSys.SignalService(serviceReloadDynamic)
-
-	// If all values were dynamic, tell the client.
 	if dynamic {
+		// Apply dynamic values.
+		if err := applyDynamicConfig(GlobalContext, objectAPI, cfg); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		globalNotificationSys.SignalService(serviceReloadDynamic)
+		// If all values were dynamic, tell the client.
 		w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue)
 	}
 	writeSuccessResponseHeadersOnly(w)
@@ -176,7 +176,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigKV")
 
-	defer logger.AuditLog(w, r, "GetConfigKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -214,7 +214,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClearConfigHistoryKV")
 
-	defer logger.AuditLog(w, r, "ClearConfigHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -251,7 +251,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RestoreConfigHistoryKV")
 
-	defer logger.AuditLog(w, r, "RestoreConfigHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -282,7 +282,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -299,7 +299,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListConfigHistoryKV")
 
-	defer logger.AuditLog(w, r, "ListConfigHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -339,7 +339,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HelpConfigKV")
 
-	defer logger.AuditLog(w, r, "HelpHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -367,7 +367,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfig")
 
-	defer logger.AuditLog(w, r, "SetConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -394,7 +394,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -424,7 +424,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfig")
 
-	defer logger.AuditLog(w, r, "GetConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {

cmd/admin-handlers-users.go

@@ -23,8 +23,10 @@ import (
 	"io/ioutil"
 	"net/http"
 	"path"
+	"sort"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
@@ -56,7 +58,7 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveUser")
 
-	defer logger.AuditLog(w, r, "RemoveUser", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
 	if objectAPI == nil {
@@ -66,7 +68,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
 
-	ok, err := globalIAMSys.IsTempUser(accessKey)
+	ok, _, err := globalIAMSys.IsTempUser(accessKey)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -94,7 +96,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListUsers")
 
-	defer logger.AuditLog(w, r, "ListUsers", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
 	if objectAPI == nil {
@@ -128,7 +130,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetUserInfo")
 
-	defer logger.AuditLog(w, r, "GetUserInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	name := vars["accessKey"]
@@ -184,7 +186,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "UpdateGroupMembers")
 
-	defer logger.AuditLog(w, r, "UpdateGroupMembers", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
 	if objectAPI == nil {
@@ -229,7 +231,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetGroup")
 
-	defer logger.AuditLog(w, r, "GetGroup", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetGroupAdminAction)
 	if objectAPI == nil {
@@ -258,7 +260,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListGroups")
 
-	defer logger.AuditLog(w, r, "ListGroups", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
 	if objectAPI == nil {
@@ -284,7 +286,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetGroupStatus")
 
-	defer logger.AuditLog(w, r, "SetGroupStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
 	if objectAPI == nil {
@@ -321,7 +323,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
 
-	defer logger.AuditLog(w, r, "SetUserStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableUserAdminAction)
 	if objectAPI == nil {
@@ -356,7 +358,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddUser")
 
-	defer logger.AuditLog(w, r, "AddUser", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	accessKey := path.Clean(vars["accessKey"])
@@ -405,6 +407,18 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if implicitPerm && !globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     accessKey,
+		Action:          iampolicy.CreateUserAdminAction,
+		ConditionValues: getConditionValues(r, "", accessKey, claims),
+		IsOwner:         owner,
+		Claims:          claims,
+		DenyOnly:        true, // check if changing password is explicitly denied.
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
@@ -426,7 +440,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
+	if err = globalIAMSys.CreateUser(accessKey, uinfo); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -444,7 +458,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddServiceAccount")
 
-	defer logger.AuditLog(w, r, "AddServiceAccount", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -483,7 +497,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		parentUser = cred.ParentUser
 	}
 
-	newCred, err := globalIAMSys.NewServiceAccount(ctx, parentUser, createReq.Policy)
+	newCred, err := globalIAMSys.NewServiceAccount(ctx, parentUser, cred.Groups, createReq.Policy)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -523,7 +537,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListServiceAccounts")
 
-	defer logger.AuditLog(w, r, "ListServiceAccounts", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -578,7 +592,7 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteServiceAccount")
 
-	defer logger.AuditLog(w, r, "DeleteServiceAccount", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -637,7 +651,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AccountInfo")
 
-	defer logger.AuditLog(w, r, "AccountInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -688,12 +702,6 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	buckets, err := objectAPI.ListBuckets(ctx)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
 	// Load the latest calculated data usage
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err != nil {
@@ -701,11 +709,34 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		logger.LogIf(ctx, err)
 	}
 
-	accountName := cred.AccessKey
-	if cred.ParentUser != "" {
-		accountName = cred.ParentUser
+	// If etcd, dns federation configured list buckets from etcd.
+	var buckets []BucketInfo
+	if globalDNSConfig != nil && globalBucketFederation {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil && !IsErrIgnored(err,
+			dns.ErrNoEntriesFound,
+			dns.ErrDomainMissing) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+		for _, dnsRecords := range dnsBuckets {
+			buckets = append(buckets, BucketInfo{
+				Name:    dnsRecords[0].Key,
+				Created: dnsRecords[0].CreationDate,
+			})
+		}
+		sort.Slice(buckets, func(i, j int) bool {
+			return buckets[i].Name < buckets[j].Name
+		})
+	} else {
+		buckets, err = objectAPI.ListBuckets(ctx)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
+	accountName := cred.AccessKey
 	policies, err := globalIAMSys.PolicyDBGet(accountName, false)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -751,7 +782,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicyV2")
 
-	defer logger.AuditLog(w, r, "InfoCannedPolicyV2", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
@@ -778,7 +809,7 @@ func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")
 
-	defer logger.AuditLog(w, r, "InfoCannedPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
@@ -802,7 +833,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPoliciesV2")
 
-	defer logger.AuditLog(w, r, "ListCannedPoliciesV2", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
@@ -836,7 +867,7 @@ func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPolicies")
 
-	defer logger.AuditLog(w, r, "ListCannedPolicies", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
@@ -870,7 +901,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveCannedPolicy")
 
-	defer logger.AuditLog(w, r, "RemoveCannedPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
 	if objectAPI == nil {
@@ -898,7 +929,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddCannedPolicy")
 
-	defer logger.AuditLog(w, r, "AddCannedPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
 	if objectAPI == nil {
@@ -950,7 +981,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
 
-	defer logger.AuditLog(w, r, "SetPolicyForUserOrGroup", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
 	if objectAPI == nil {
@@ -963,7 +994,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 	isGroup := vars["isGroup"] == "true"
 
 	if !isGroup {
-		ok, err := globalIAMSys.IsTempUser(entityName)
+		ok, _, err := globalIAMSys.IsTempUser(entityName)
 		if err != nil && err != errNoSuchUser {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return

cmd/admin-handlers.go

@@ -42,6 +42,7 @@ import (
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/bandwidth"
+	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/handlers"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -78,7 +79,7 @@ func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, mode string) (
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerUpdate")
 
-	defer logger.AuditLog(w, r, "ServerUpdate", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
 	if objectAPI == nil {
@@ -187,7 +188,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "Service")
 
-	defer logger.AuditLog(w, r, "Service", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	action := vars["action"]
@@ -258,6 +259,7 @@ type ServerHTTPAPIStats struct {
 // ServerHTTPStats holds all type of http operations performed to/from the server
 // including their average execution time.
 type ServerHTTPStats struct {
+	S3RequestsInQueue int32              `json:"s3RequestsInQueue"`
 	CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"`
 	TotalS3Requests   ServerHTTPAPIStats `json:"totalS3Requests"`
 	TotalS3Errors     ServerHTTPAPIStats `json:"totalS3Errors"`
@@ -284,7 +286,7 @@ type ServerInfo struct {
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StorageInfo")
 
-	defer logger.AuditLog(w, r, "StorageInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
 	if objectAPI == nil {
@@ -295,7 +297,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 	storageInfo, _ := objectAPI.StorageInfo(ctx)
 
 	// Collect any disk healing.
-	healing, _ := getAggregatedBackgroundHealState(ctx)
+	healing, _ := getAggregatedBackgroundHealState(ctx, nil)
 	healDisks := make(map[string]struct{}, len(healing.HealDisks))
 	for _, disk := range healing.HealDisks {
 		healDisks[disk] = struct{}{}
@@ -327,7 +329,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DataUsageInfo")
 
-	defer logger.AuditLog(w, r, "DataUsageInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
@@ -403,11 +405,50 @@ type PeerLocks struct {
 	Locks map[string][]lockRequesterInfo
 }
 
+// ForceUnlockHandler force unlocks requested resource
+func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ForceUnlock")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ForceUnlockAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	z, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+
+	var args dsync.LockArgs
+	lockersMap := make(map[string]dsync.NetLocker)
+	for _, path := range strings.Split(vars["paths"], ",") {
+		if path == "" {
+			continue
+		}
+		args.Resources = append(args.Resources, path)
+		lockers, _ := z.serverPools[0].getHashedSet(path).getLockers()
+		for _, locker := range lockers {
+			if locker != nil {
+				lockersMap[locker.String()] = locker
+			}
+		}
+	}
+
+	for _, locker := range lockersMap {
+		locker.ForceUnlock(ctx, args)
+	}
+}
+
 // TopLocksHandler Get list of locks in use
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "TopLocks")
 
-	defer logger.AuditLog(w, r, "TopLocks", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
 	if objectAPI == nil {
@@ -459,7 +500,7 @@ type StartProfilingResult struct {
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
-	defer logger.AuditLog(w, r, "StartProfiling", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
@@ -564,7 +605,7 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
-	defer logger.AuditLog(w, r, "DownloadProfiling", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
@@ -665,7 +706,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "Heal")
 
-	defer logger.AuditLog(w, r, "Heal", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
@@ -820,16 +861,14 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	keepConnLive(w, r, respCh)
 }
 
-func getAggregatedBackgroundHealState(ctx context.Context) (madmin.BgHealState, error) {
-	var bgHealStates []madmin.BgHealState
-
-	localHealState, ok := getLocalBackgroundHealStatus()
-	if !ok {
-		return madmin.BgHealState{}, errServerNotInitialized
-	}
-
+// getAggregatedBackgroundHealState returns the heal state of disks.
+// If no ObjectLayer is provided no set status is returned.
+func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmin.BgHealState, error) {
 	// Get local heal status first
-	bgHealStates = append(bgHealStates, localHealState)
+	bgHealStates, ok := getBackgroundHealStatus(ctx, o)
+	if !ok {
+		return bgHealStates, errServerNotInitialized
+	}
 
 	if globalIsDistErasure {
 		// Get heal status from other peers
@@ -844,39 +883,16 @@ func getAggregatedBackgroundHealState(ctx context.Context) (madmin.BgHealState,
 		if errCount == len(nerrs) {
 			return madmin.BgHealState{}, fmt.Errorf("all remote servers failed to report heal status, cluster is unhealthy")
 		}
-		bgHealStates = append(bgHealStates, peersHealStates...)
+		bgHealStates.Merge(peersHealStates...)
 	}
 
-	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{
-		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
-		LastHealActivity:  bgHealStates[0].LastHealActivity,
-		NextHealRound:     bgHealStates[0].NextHealRound,
-		HealDisks:         bgHealStates[0].HealDisks,
-	}
-
-	bgHealStates = bgHealStates[1:]
-
-	for _, state := range bgHealStates {
-		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		aggregatedHealStateResult.HealDisks = append(aggregatedHealStateResult.HealDisks, state.HealDisks...)
-		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
-			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
-			// The node which has the last heal activity means its
-			// is the node that is orchestrating self healing operations,
-			// which also means it is the same node which decides when
-			// the next self healing operation will be done.
-			aggregatedHealStateResult.NextHealRound = state.NextHealRound
-		}
-	}
-
-	return aggregatedHealStateResult, nil
+	return bgHealStates, nil
 }
 
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealBackgroundStatus")
 
-	defer logger.AuditLog(w, r, "HealBackgroundStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
@@ -889,7 +905,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context())
+	aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context(), objectAPI)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1097,7 +1113,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ConsoleLog")
 
-	defer logger.AuditLog(w, r, "ConsoleLog", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
 	if objectAPI == nil {
@@ -1168,7 +1184,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 // KMSCreateKeyHandler - POST /minio/admin/v3/kms/key/create?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSCreateKey")
-	defer logger.AuditLog(w, r, "KMSCreateKey", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
 	if objectAPI == nil {
@@ -1191,7 +1207,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSKeyStatus")
 
-	defer logger.AuditLog(w, r, "KMSKeyStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
@@ -1264,7 +1280,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealthInfo")
 
-	defer logger.AuditLog(w, r, "HealthInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1307,8 +1323,10 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
 	defer cancel()
 
+	var err error
 	nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress")
-	if err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
+	ctx, err = nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline))
+	if err != nil { // returns a locked lock
 		errResp(err)
 		return
 	}
@@ -1442,7 +1460,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "BandwidthMonitor")
 
-	defer logger.AuditLog(w, r, "BandwidthMonitor", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.BandwidthMonitorAction, "")
@@ -1495,7 +1513,7 @@ func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
 
-	defer logger.AuditLog(w, r, "ServerInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
@@ -1504,19 +1522,19 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	vault := fetchVaultStatus()
+	kmsStat := fetchKMSStatus()
 
 	ldap := madmin.LDAP{}
 	if globalLDAPConfig.Enabled {
 		ldapConn, err := globalLDAPConfig.Connect()
 		if err != nil {
-			ldap.Status = "offline"
+			ldap.Status = string(madmin.ItemOffline)
 		} else if ldapConn == nil {
 			ldap.Status = "Not Configured"
 		} else {
 			// Close ldap connection to avoid leaks.
 			ldapConn.Close()
-			ldap.Status = "online"
+			ldap.Status = string(madmin.ItemOnline)
 		}
 	}
 
@@ -1525,12 +1543,14 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	// Get the notification target info
 	notifyTarget := fetchLambdaInfo()
 
-	server := getLocalServerProperty(globalEndpoints, r)
+	local := getLocalServerProperty(globalEndpoints, r)
 	servers := globalNotificationSys.ServerInfo()
-	servers = append(servers, server)
+	servers = append(servers, local)
+
+	assignPoolNumbers(servers)
 
 	var backend interface{}
-	mode := madmin.ObjectLayerInitializing
+	mode := madmin.ItemInitializing
 
 	buckets := madmin.Buckets{}
 	objects := madmin.Objects{}
@@ -1538,18 +1558,22 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	objectAPI := newObjectLayerFn()
 	if objectAPI != nil {
-		mode = madmin.ObjectLayerOnline
+		mode = madmin.ItemOnline
 		// Load data usage
 		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 		if err == nil {
 			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
 			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
 			usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
+		} else {
+			buckets = madmin.Buckets{Error: err.Error()}
+			objects = madmin.Objects{Error: err.Error()}
+			usage = madmin.Usage{Error: err.Error()}
 		}
 
 		// Fetching the backend information
 		backendInfo := objectAPI.BackendInfo()
-		if backendInfo.Type == BackendType(madmin.Erasure) {
+		if backendInfo.Type == madmin.Erasure {
 			// Calculate the number of online/offline disks of all nodes
 			var allDisks []madmin.Disk
 			for _, s := range servers {
@@ -1561,9 +1585,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 				Type:             madmin.ErasureType,
 				OnlineDisks:      onlineDisks.Sum(),
 				OfflineDisks:     offlineDisks.Sum(),
-				StandardSCData:   backendInfo.StandardSCData,
 				StandardSCParity: backendInfo.StandardSCParity,
-				RRSCData:         backendInfo.RRSCData,
 				RRSCParity:       backendInfo.RRSCParity,
 			}
 		} else {
@@ -1575,7 +1597,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	domain := globalDomainNames
 	services := madmin.Services{
-		Vault:         vault,
+		KMS:           kmsStat,
 		LDAP:          ldap,
 		Logger:        log,
 		Audit:         audit,
@@ -1583,7 +1605,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	infoMsg := madmin.InfoMessage{
-		Mode:         mode,
+		Mode:         string(mode),
 		Domain:       domain,
 		Region:       globalServerRegion,
 		SQSARN:       globalNotificationSys.GetARNList(false),
@@ -1608,6 +1630,22 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
+func assignPoolNumbers(servers []madmin.ServerProperties) {
+	for i := range servers {
+		for idx, ge := range globalEndpoints {
+			for _, endpoint := range ge.Endpoints {
+				if servers[i].Endpoint == endpoint.Host {
+					servers[i].PoolNumber = idx + 1
+				} else if host, err := xnet.ParseHost(servers[i].Endpoint); err == nil {
+					if host.Name == endpoint.Hostname() {
+						servers[i].PoolNumber = idx + 1
+					}
+				}
+			}
+		}
+	}
+}
+
 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
@@ -1617,9 +1655,9 @@ func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 		active, _ := tgt.IsActive()
 		targetID := tgt.ID()
 		if active {
-			targetIDStatus[targetID.ID] = madmin.Status{Status: "Online"}
+			targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOnline)}
 		} else {
-			targetIDStatus[targetID.ID] = madmin.Status{Status: "Offline"}
+			targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOffline)}
 		}
 		list := lambdaMap[targetID.Name]
 		list = append(list, targetIDStatus)
@@ -1631,9 +1669,9 @@ func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 		active, _ := tgt.IsActive()
 		targetID := tgt.ID()
 		if active {
-			targetIDStatus[targetID.ID] = madmin.Status{Status: "Online"}
+			targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOnline)}
 		} else {
-			targetIDStatus[targetID.ID] = madmin.Status{Status: "Offline"}
+			targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOffline)}
 		}
 		list := lambdaMap[targetID.Name]
 		list = append(list, targetIDStatus)
@@ -1651,47 +1689,46 @@ func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 	return notify
 }
 
-// fetchVaultStatus fetches Vault Info
-func fetchVaultStatus() madmin.Vault {
-	vault := madmin.Vault{}
+// fetchKMSStatus fetches KMS-related status information.
+func fetchKMSStatus() madmin.KMS {
+	kmsStat := madmin.KMS{}
 	if GlobalKMS == nil {
-		vault.Status = "disabled"
-		return vault
+		kmsStat.Status = "disabled"
+		return kmsStat
 	}
 	keyID := GlobalKMS.DefaultKeyID()
 	kmsInfo := GlobalKMS.Info()
-
 	if len(kmsInfo.Endpoints) == 0 {
-		vault.Status = "KMS configured using master key"
-		return vault
+		kmsStat.Status = "KMS configured using master key"
+		return kmsStat
 	}
 
 	if err := checkConnection(kmsInfo.Endpoints[0], 15*time.Second); err != nil {
-		vault.Status = "offline"
+		kmsStat.Status = string(madmin.ItemOffline)
 	} else {
-		vault.Status = "online"
+		kmsStat.Status = string(madmin.ItemOnline)
 
 		kmsContext := crypto.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
 		// 1. Generate a new key using the KMS.
 		key, sealedKey, err := GlobalKMS.GenerateKey(keyID, kmsContext)
 		if err != nil {
-			vault.Encrypt = fmt.Sprintf("Encryption failed: %v", err)
+			kmsStat.Encrypt = fmt.Sprintf("Encryption failed: %v", err)
 		} else {
-			vault.Encrypt = "Ok"
+			kmsStat.Encrypt = "success"
 		}
 
 		// 2. Verify that we can indeed decrypt the (encrypted) key
 		decryptedKey, err := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
 		switch {
 		case err != nil:
-			vault.Decrypt = fmt.Sprintf("Decryption failed: %v", err)
+			kmsStat.Decrypt = fmt.Sprintf("Decryption failed: %v", err)
 		case subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1:
-			vault.Decrypt = "Decryption failed: decrypted key does not match generated key"
+			kmsStat.Decrypt = "Decryption failed: decrypted key does not match generated key"
 		default:
-			vault.Decrypt = "Ok"
+			kmsStat.Decrypt = "success"
 		}
 	}
-	return vault
+	return kmsStat
 }
 
 // fetchLoggerDetails return log info
@@ -1704,11 +1741,11 @@ func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
 			err := checkConnection(target.Endpoint(), 15*time.Second)
 			if err == nil {
 				mapLog := make(map[string]madmin.Status)
-				mapLog[tgt] = madmin.Status{Status: "Online"}
+				mapLog[tgt] = madmin.Status{Status: string(madmin.ItemOnline)}
 				loggerInfo = append(loggerInfo, mapLog)
 			} else {
 				mapLog := make(map[string]madmin.Status)
-				mapLog[tgt] = madmin.Status{Status: "offline"}
+				mapLog[tgt] = madmin.Status{Status: string(madmin.ItemOffline)}
 				loggerInfo = append(loggerInfo, mapLog)
 			}
 		}
@@ -1720,11 +1757,11 @@ func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
 			err := checkConnection(target.Endpoint(), 15*time.Second)
 			if err == nil {
 				mapAudit := make(map[string]madmin.Status)
-				mapAudit[tgt] = madmin.Status{Status: "Online"}
+				mapAudit[tgt] = madmin.Status{Status: string(madmin.ItemOnline)}
 				auditloggerInfo = append(auditloggerInfo, mapAudit)
 			} else {
 				mapAudit := make(map[string]madmin.Status)
-				mapAudit[tgt] = madmin.Status{Status: "Offline"}
+				mapAudit[tgt] = madmin.Status{Status: string(madmin.ItemOffline)}
 				auditloggerInfo = append(auditloggerInfo, mapAudit)
 			}
 		}

cmd/admin-handlers_test.go

@@ -66,7 +66,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 	// Initialize boot time
 	globalBootTime = UTCNow()
 
-	globalEndpoints = mustGetZoneEndpoints(erasureDirs...)
+	globalEndpoints = mustGetPoolEndpoints(erasureDirs...)
 
 	newAllSubsystems()
 
@@ -97,16 +97,9 @@ func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error)
 	if err != nil {
 		return nil, nil, err
 	}
-	endpoints := mustGetNewEndpoints(erasureDirs...)
-	storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "")
-	if err != nil {
-		removeRoots(erasureDirs)
-		return nil, nil, err
-	}
-
+	endpoints := mustGetPoolEndpoints(erasureDirs...)
 	globalPolicySys = NewPolicySys()
-	objLayer := &erasureServerPools{serverPools: make([]*erasureSets, 1)}
-	objLayer.serverPools[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
+	objLayer, err := newErasureServerPools(ctx, endpoints)
 	if err != nil {
 		return nil, nil, err
 	}

cmd/admin-heal-ops.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
-	"strings"
 	"sync"
 	"time"
 
@@ -90,8 +89,9 @@ type allHealState struct {
 	sync.RWMutex
 
 	// map of heal path to heal sequence
-	healSeqMap     map[string]*healSequence
+	healSeqMap     map[string]*healSequence // Indexed by endpoint
 	healLocalDisks map[Endpoint]struct{}
+	healStatus     map[string]healingTracker // Indexed by disk ID
 }
 
 // newHealState - initialize global heal state management
@@ -99,6 +99,7 @@ func newHealState(cleanup bool) *allHealState {
 	hstate := &allHealState{
 		healSeqMap:     make(map[string]*healSequence),
 		healLocalDisks: map[Endpoint]struct{}{},
+		healStatus:     make(map[string]healingTracker),
 	}
 	if cleanup {
 		go hstate.periodicHealSeqsClean(GlobalContext)
@@ -113,7 +114,56 @@ func (ahs *allHealState) healDriveCount() int {
 	return len(ahs.healLocalDisks)
 }
 
-func (ahs *allHealState) getHealLocalDisks() Endpoints {
+func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	for _, ep := range healLocalDisks {
+		delete(ahs.healLocalDisks, ep)
+	}
+	for id, disk := range ahs.healStatus {
+		for _, ep := range healLocalDisks {
+			if disk.Endpoint == ep.String() {
+				delete(ahs.healStatus, id)
+			}
+		}
+	}
+}
+
+// updateHealStatus will update the heal status.
+func (ahs *allHealState) updateHealStatus(tracker *healingTracker) {
+	ahs.Lock()
+	defer ahs.Unlock()
+	ahs.healStatus[tracker.ID] = *tracker
+}
+
+// Sort by zone, set and disk index
+func sortDisks(disks []madmin.Disk) {
+	sort.Slice(disks, func(i, j int) bool {
+		a, b := &disks[i], &disks[j]
+		if a.PoolIndex != b.PoolIndex {
+			return a.PoolIndex < b.PoolIndex
+		}
+		if a.SetIndex != b.SetIndex {
+			return a.SetIndex < b.SetIndex
+		}
+		return a.DiskIndex < b.DiskIndex
+	})
+}
+
+// getLocalHealingDisks returns local healing disks indexed by endpoint.
+func (ahs *allHealState) getLocalHealingDisks() map[string]madmin.HealingDisk {
+	ahs.RLock()
+	defer ahs.RUnlock()
+	dst := make(map[string]madmin.HealingDisk, len(ahs.healStatus))
+	for _, v := range ahs.healStatus {
+		dst[v.Endpoint] = v.toHealingDisk()
+	}
+
+	return dst
+}
+
+func (ahs *allHealState) getHealLocalDiskEndpoints() Endpoints {
 	ahs.RLock()
 	defer ahs.RUnlock()
 
@@ -124,15 +174,6 @@ func (ahs *allHealState) getHealLocalDisks() Endpoints {
 	return endpoints
 }
 
-func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) {
-	ahs.Lock()
-	defer ahs.Unlock()
-
-	for _, ep := range healLocalDisks {
-		delete(ahs.healLocalDisks, ep)
-	}
-}
-
 func (ahs *allHealState) pushHealLocalDisks(healLocalDisks ...Endpoint) {
 	ahs.Lock()
 	defer ahs.Unlock()
@@ -662,6 +703,13 @@ func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 	}
 }
 
+func (h *healSequence) logHeal(healType madmin.HealItemType) {
+	h.mutex.Lock()
+	h.scannedItemsMap[healType]++
+	h.lastHealActivity = UTCNow()
+	h.mutex.Unlock()
+}
+
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
 	globalHealConfigMu.Lock()
 	opts := globalHealConfig
@@ -750,16 +798,19 @@ func (h *healSequence) healItemsFromSourceCh() error {
 			if !ok {
 				return nil
 			}
+
 			var itemType madmin.HealItemType
-			switch {
-			case source.bucket == nopHeal:
+			switch source.bucket {
+			case nopHeal:
 				continue
-			case source.bucket == SlashSeparator:
+			case SlashSeparator:
 				itemType = madmin.HealItemMetadata
-			case source.bucket != "" && source.object == "":
-				itemType = madmin.HealItemBucket
 			default:
-				itemType = madmin.HealItemObject
+				if source.object == "" {
+					itemType = madmin.HealItemBucket
+				} else {
+					itemType = madmin.HealItemObject
+				}
 			}
 
 			if err := h.queueHealTask(source, itemType); err != nil {
@@ -831,11 +882,6 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 				return errHealStopSignalled
 			}
 
-			// Skip metacache entries healing
-			if strings.HasPrefix(object, "buckets/.minio.sys/.metacache/") {
-				return nil
-			}
-
 			err := h.queueHealTask(healSource{
 				bucket:    bucket,
 				object:    object,

cmd/admin-router.go

@@ -110,10 +110,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			// -- IAM APIs --
 
 			// Add policy IAM
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceAll(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
 
 			// Add user IAM
-
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(httpTraceAll(adminAPI.AccountInfoHandler))
 
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
@@ -189,10 +188,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 				httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 		}
 
-		// -- Top APIs --
-		// Top locks
 		if globalIsDistErasure {
+			// Top locks
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+			// Force unlocks paths
+			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/force-unlock").
+				Queries("paths", "{paths:.*}").HandlerFunc(httpTraceHdrs(adminAPI.ForceUnlockHandler))
 		}
 
 		// HTTP Trace

cmd/admin-server-info.go

@@ -17,8 +17,11 @@
 package cmd
 
 import (
+	"context"
 	"net/http"
+	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -39,33 +42,38 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 			}
 			if endpoint.IsLocal {
 				// Only proceed for local endpoints
-				network[nodeName] = "online"
+				network[nodeName] = string(madmin.ItemOnline)
 				localEndpoints = append(localEndpoints, endpoint)
 				continue
 			}
 			_, present := network[nodeName]
 			if !present {
-				if err := isServerResolvable(endpoint); err == nil {
-					network[nodeName] = "online"
+				if err := isServerResolvable(endpoint, 2*time.Second); err == nil {
+					network[nodeName] = string(madmin.ItemOnline)
 				} else {
-					network[nodeName] = "offline"
+					network[nodeName] = string(madmin.ItemOffline)
+					// log once the error
+					logger.LogOnceIf(context.Background(), err, nodeName)
 				}
 			}
 		}
 	}
 
-	localDisks, _ := initStorageDisksWithErrors(localEndpoints)
-	defer closeStorageDisks(localDisks)
-
-	storageInfo, _ := getStorageInfo(localDisks, localEndpoints.GetAllStrings())
-
-	return madmin.ServerProperties{
-		State:    "ok",
+	props := madmin.ServerProperties{
+		State:    string(madmin.ItemInitializing),
 		Endpoint: addr,
 		Uptime:   UTCNow().Unix() - globalBootTime.Unix(),
 		Version:  Version,
 		CommitID: CommitID,
 		Network:  network,
-		Disks:    storageInfo.Disks,
 	}
+
+	objLayer := newObjectLayerFn()
+	if objLayer != nil {
+		storageInfo, _ := objLayer.LocalStorageInfo(GlobalContext)
+		props.State = string(madmin.ItemOnline)
+		props.Disks = storageInfo.Disks
+	}
+
+	return props
 }

cmd/api-errors.go

@@ -87,6 +87,7 @@ const (
 	ErrInvalidMaxUploads
 	ErrInvalidMaxParts
 	ErrInvalidPartNumberMarker
+	ErrInvalidPartNumber
 	ErrInvalidRequestBody
 	ErrInvalidCopySource
 	ErrInvalidMetadataDirective
@@ -437,6 +438,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Argument partNumberMarker must be an integer.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidPartNumber: {
+		Code:           "InvalidPartNumber",
+		Description:    "The requested partnumber is not satisfiable",
+		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
+	},
 	ErrInvalidPolicyDocument: {
 		Code:           "InvalidPolicyDocument",
 		Description:    "The content of the form does not meet the conditions specified in the policy document.",
@@ -2119,6 +2125,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 				HTTPStatusCode: e.Response().StatusCode,
 			}
 			// Add more Gateway SDKs here if any in future.
+		default:
+			apiErr = APIError{
+				Code:           apiErr.Code,
+				Description:    fmt.Sprintf("%s: cause(%v)", apiErr.Description, err),
+				HTTPStatusCode: apiErr.HTTPStatusCode,
+			}
 		}
 	}
 

cmd/api-headers.go

@@ -133,18 +133,20 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		}
 
 		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+		if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 			continue
 		}
+
 		var isSet bool
 		for _, userMetadataPrefix := range userMetadataKeyPrefixes {
-			if !strings.HasPrefix(k, userMetadataPrefix) {
+			if !strings.HasPrefix(strings.ToLower(k), strings.ToLower(userMetadataPrefix)) {
 				continue
 			}
 			w.Header()[strings.ToLower(k)] = []string{v}
 			isSet = true
 			break
 		}
+
 		if !isSet {
 			w.Header().Set(k, v)
 		}
@@ -156,16 +158,16 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		return err
 	}
 
-	if opts.PartNumber > 0 {
-		rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
-	}
-
 	// For providing ranged content
 	start, rangeLen, err = rs.GetOffsetLength(totalObjectSize)
 	if err != nil {
 		return err
 	}
 
+	if rs == nil && opts.PartNumber > 0 {
+		rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
+	}
+
 	// Set content length.
 	w.Header().Set(xhttp.ContentLength, strconv.FormatInt(rangeLen, 10))
 	if rs != nil {
@@ -177,21 +179,25 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	if objInfo.VersionID != "" {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}
+
 	if objInfo.ReplicationStatus.String() != "" {
 		w.Header()[xhttp.AmzBucketReplicationStatus] = []string{objInfo.ReplicationStatus.String()}
 	}
+
 	if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil {
-		ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{
-			Name:         objInfo.Name,
-			UserTags:     objInfo.UserTags,
-			VersionID:    objInfo.VersionID,
-			ModTime:      objInfo.ModTime,
-			IsLatest:     objInfo.IsLatest,
-			DeleteMarker: objInfo.DeleteMarker,
-		})
-		if !expiryTime.IsZero() {
-			w.Header()[xhttp.AmzExpiration] = []string{
-				fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID),
+		if opts.VersionID == "" {
+			if ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{
+				Name:             objInfo.Name,
+				UserTags:         objInfo.UserTags,
+				VersionID:        objInfo.VersionID,
+				ModTime:          objInfo.ModTime,
+				IsLatest:         objInfo.IsLatest,
+				DeleteMarker:     objInfo.DeleteMarker,
+				SuccessorModTime: objInfo.SuccessorModTime,
+			}); !expiryTime.IsZero() {
+				w.Header()[xhttp.AmzExpiration] = []string{
+					fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID),
+				}
 			}
 		}
 		if objInfo.TransitionStatus == lifecycle.TransitionComplete {

cmd/api-resources.go

@@ -36,8 +36,8 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
 		maxkeys = maxObjectList
 	}
 
-	prefix = values.Get("prefix")
-	marker = values.Get("marker")
+	prefix = trimLeadingSlash(values.Get("prefix"))
+	marker = trimLeadingSlash(values.Get("marker"))
 	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	return
@@ -56,8 +56,8 @@ func getListBucketObjectVersionsArgs(values url.Values) (prefix, marker, delimit
 		maxkeys = maxObjectList
 	}
 
-	prefix = values.Get("prefix")
-	marker = values.Get("key-marker")
+	prefix = trimLeadingSlash(values.Get("prefix"))
+	marker = trimLeadingSlash(values.Get("key-marker"))
 	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	versionIDMarker = values.Get("version-id-marker")
@@ -86,8 +86,8 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 		maxkeys = maxObjectList
 	}
 
-	prefix = values.Get("prefix")
-	startAfter = values.Get("start-after")
+	prefix = trimLeadingSlash(values.Get("prefix"))
+	startAfter = trimLeadingSlash(values.Get("start-after"))
 	delimiter = values.Get("delimiter")
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
@@ -117,8 +117,8 @@ func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadID
 		maxUploads = maxUploadsList
 	}
 
-	prefix = values.Get("prefix")
-	keyMarker = values.Get("key-marker")
+	prefix = trimLeadingSlash(values.Get("prefix"))
+	keyMarker = trimLeadingSlash(values.Get("key-marker"))
 	uploadIDMarker = values.Get("upload-id-marker")
 	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")

cmd/api-response.go

@@ -48,6 +48,12 @@ type LocationResponse struct {
 	Location string   `xml:",chardata"`
 }
 
+// PolicyStatus captures information returned by GetBucketPolicyStatusHandler
+type PolicyStatus struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PolicyStatus" json:"-"`
+	IsPublic string
+}
+
 // ListVersionsResponse - format for list bucket versions response.
 type ListVersionsResponse struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult" json:"-"`
@@ -410,9 +416,11 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
 func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	listbuckets := make([]Bucket, 0, len(buckets))
 	var data = ListBucketsResponse{}
-	var owner = Owner{}
+	var owner = Owner{
+		ID:          globalMinioDefaultOwnerID,
+		DisplayName: "minio",
+	}
 
-	owner.ID = globalMinioDefaultOwnerID
 	for _, bucket := range buckets {
 		var listbucket = Bucket{}
 		listbucket.Name = bucket.Name
@@ -429,10 +437,12 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
-	var owner = Owner{}
+	var owner = Owner{
+		ID:          globalMinioDefaultOwnerID,
+		DisplayName: "minio",
+	}
 	var data = ListVersionsResponse{}
 
-	owner.ID = globalMinioDefaultOwnerID
 	for _, object := range resp.Objects {
 		var content = ObjectVersion{}
 		if object.Name == "" {
@@ -485,10 +495,12 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
 func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
 	contents := make([]Object, 0, len(resp.Objects))
-	var owner = Owner{}
+	var owner = Owner{
+		ID:          globalMinioDefaultOwnerID,
+		DisplayName: "minio",
+	}
 	var data = ListObjectsResponse{}
 
-	owner.ID = globalMinioDefaultOwnerID
 	for _, object := range resp.Objects {
 		var content = Object{}
 		if object.Name == "" {
@@ -532,12 +544,11 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
 func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
 	contents := make([]Object, 0, len(objects))
-	var owner = Owner{}
-	var data = ListObjectsV2Response{}
-
-	if fetchOwner {
-		owner.ID = globalMinioDefaultOwnerID
+	var owner = Owner{
+		ID:          globalMinioDefaultOwnerID,
+		DisplayName: "minio",
 	}
+	var data = ListObjectsV2Response{}
 
 	for _, object := range objects {
 		var content = Object{}
@@ -565,7 +576,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 					continue
 				}
 				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-				if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 					continue
 				}
 				content.UserMetadata[k] = v
@@ -639,8 +650,16 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 	listPartsResponse.Key = s3EncodeName(partsInfo.Object, encodingType)
 	listPartsResponse.UploadID = partsInfo.UploadID
 	listPartsResponse.StorageClass = globalMinioDefaultStorageClass
-	listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID
-	listPartsResponse.Owner.ID = globalMinioDefaultOwnerID
+
+	// Dumb values not meaningful
+	listPartsResponse.Initiator = Initiator{
+		ID:          globalMinioDefaultOwnerID,
+		DisplayName: globalMinioDefaultOwnerID,
+	}
+	listPartsResponse.Owner = Owner{
+		ID:          globalMinioDefaultOwnerID,
+		DisplayName: globalMinioDefaultOwnerID,
+	}
 
 	listPartsResponse.MaxParts = partsInfo.MaxParts
 	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker

cmd/api-router.go

@@ -185,6 +185,10 @@ func registerAPIRouter(router *mux.Router) {
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("deleteobject", maxClients(httpTraceAll(api.DeleteObjectHandler))))
 
+		// PostRestoreObject
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
+			collectAPIStats("restoreobject", maxClients(httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "")
+
 		/// Bucket operations
 		// GetBucketLocation
 		bucket.Methods(http.MethodGet).HandlerFunc(
@@ -262,9 +266,9 @@ func registerAPIRouter(router *mux.Router) {
 		// ListObjectVersions
 		bucket.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectversions", maxClients(httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "")
-		// ListObjectsV1 (Legacy)
+		// GetBucketPolicyStatus
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectsv1", maxClients(httpTraceAll(api.ListObjectsV1Handler))))
+			collectAPIStats("getpolicystatus", maxClients(httpTraceAll(api.GetBucketPolicyStatusHandler)))).Queries("policyStatus", "")
 		// PutBucketLifecycle
 		bucket.Methods(http.MethodPut).HandlerFunc(
 			collectAPIStats("putbucketlifecycle", maxClients(httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "")
@@ -320,9 +324,9 @@ func registerAPIRouter(router *mux.Router) {
 		// DeleteBucket
 		bucket.Methods(http.MethodDelete).HandlerFunc(
 			collectAPIStats("deletebucket", maxClients(httpTraceAll(api.DeleteBucketHandler))))
-		// PostRestoreObject
-		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("restoreobject", maxClients(httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "")
+		// ListObjectsV1 (Legacy)
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			collectAPIStats("listobjectsv1", maxClients(httpTraceAll(api.ListObjectsV1Handler))))
 	}
 
 	/// Root operation

cmd/auth-handler.go

@@ -36,6 +36,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/etag"
 	"github.com/minio/minio/pkg/hash"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 )
@@ -185,12 +186,12 @@ func getSessionToken(r *http.Request) (token string) {
 // Fetch claims in the security token returned by the client, doesn't return
 // errors - upon errors the returned claims map will be empty.
 func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
-	claims, _ := getClaimsFromToken(r, getSessionToken(r))
+	claims, _ := getClaimsFromToken(getSessionToken(r))
 	return claims
 }
 
 // Fetch claims in the security token returned by the client.
-func getClaimsFromToken(r *http.Request, token string) (map[string]interface{}, error) {
+func getClaimsFromToken(token string) (map[string]interface{}, error) {
 	claims := xjwt.NewMapClaims()
 	if token == "" {
 		return claims.Map(), nil
@@ -237,7 +238,7 @@ func getClaimsFromToken(r *http.Request, token string) (map[string]interface{},
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
 			// something is malforming the request sent by client.
-			logger.LogIf(r.Context(), err, logger.Application)
+			logger.LogIf(GlobalContext, err, logger.Application)
 			return nil, errAuthentication
 		}
 		claims.MapClaims[iampolicy.SessionPolicyName] = string(spBytes)
@@ -258,7 +259,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 	if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
 		return nil, ErrInvalidToken
 	}
-	claims, err := getClaimsFromToken(r, token)
+	claims, err := getClaimsFromToken(token)
 	if err != nil {
 		return nil, toAPIErrorCode(r.Context(), err)
 	}
@@ -430,19 +431,14 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 		return errCode
 	}
 
-	var (
-		err                       error
-		contentMD5, contentSHA256 []byte
-	)
-
-	// Extract 'Content-Md5' if present.
-	contentMD5, err = checkValidMD5(r.Header)
+	clientETag, err := etag.FromContentMD5(r.Header)
 	if err != nil {
 		return ErrInvalidDigest
 	}
 
 	// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
 	// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
+	var contentSHA256 []byte
 	if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
 		if sha256Sum, ok := r.URL.Query()[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
 			contentSHA256, err = hex.DecodeString(sha256Sum[0])
@@ -459,8 +455,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 
 	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
 	// The verification happens implicit during reading.
-	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5),
-		hex.EncodeToString(contentSHA256), -1, globalCLIContext.StrictS3Compat)
+	reader, err := hash.NewReader(r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}
@@ -498,6 +493,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 			// Validate Authorization header if its valid for JWT request.
 			if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
 				w.WriteHeader(http.StatusUnauthorized)
+				w.Write([]byte(authErr.Error()))
 				return
 			}
 			h.ServeHTTP(w, r)

cmd/background-heal-ops.go

@@ -61,7 +61,6 @@ func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
 	}
 
 	// At max 10 attempts to wait with 100 millisecond interval before proceeding
-	waitCount := 10
 	waitTick := 100 * time.Millisecond
 
 	// Bucket notification and http trace are not costly, it is okay to ignore them
@@ -70,14 +69,21 @@ func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
 		return maxIO + int(globalHTTPListen.NumSubscribers()) + int(globalHTTPTrace.NumSubscribers())
 	}
 
+	tmpMaxWait := maxWait
 	if httpServer := newHTTPServerFn(); httpServer != nil {
 		// Any requests in progress, delay the heal.
 		for httpServer.GetRequestCount() >= maxIOFn() {
-			time.Sleep(waitTick)
-			waitCount--
-			if waitCount == 0 {
+			if tmpMaxWait > 0 {
+				if tmpMaxWait < waitTick {
+					time.Sleep(tmpMaxWait)
+				} else {
+					time.Sleep(waitTick)
+				}
+				tmpMaxWait = tmpMaxWait - waitTick
+			}
+			if tmpMaxWait <= 0 {
 				if intDataUpdateTracker.debug {
-					logger.Info("waitForLowHTTPReq: waited %d times, resuming", waitCount)
+					logger.Info("waitForLowHTTPReq: waited max %s, resuming", maxWait)
 				}
 				break
 			}
@@ -91,20 +97,22 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 		select {
 		case task, ok := <-h.tasks:
 			if !ok {
-				break
+				return
 			}
 
 			var res madmin.HealResultItem
 			var err error
-			switch {
-			case task.bucket == nopHeal:
+			switch task.bucket {
+			case nopHeal:
 				continue
-			case task.bucket == SlashSeparator:
+			case SlashSeparator:
 				res, err = healDiskFormat(ctx, objAPI, task.opts)
-			case task.bucket != "" && task.object == "":
-				res, err = objAPI.HealBucket(ctx, task.bucket, task.opts)
-			case task.bucket != "" && task.object != "":
-				res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
+			default:
+				if task.object == "" {
+					res, err = objAPI.HealBucket(ctx, task.bucket, task.opts)
+				} else {
+					res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
+				}
 			}
 			task.responseCh <- healResult{result: res, err: err}
 

cmd/background-newdisks-heal-ops.go

@@ -17,15 +17,23 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"sort"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/console"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 const (
@@ -34,10 +42,200 @@ const (
 )
 
 //go:generate msgp -file $GOFILE -unexported
-type healingTracker struct {
-	ID string
 
-	// future add more tracking capabilities
+// healingTracker is used to persist healing information during a heal.
+type healingTracker struct {
+	disk StorageAPI `msg:"-"`
+
+	ID            string
+	PoolIndex     int
+	SetIndex      int
+	DiskIndex     int
+	Path          string
+	Endpoint      string
+	Started       time.Time
+	LastUpdate    time.Time
+	ObjectsHealed uint64
+	ObjectsFailed uint64
+	BytesDone     uint64
+	BytesFailed   uint64
+
+	// Last object scanned.
+	Bucket string `json:"-"`
+	Object string `json:"-"`
+
+	// Numbers when current bucket started healing,
+	// for resuming with correct numbers.
+	ResumeObjectsHealed uint64 `json:"-"`
+	ResumeObjectsFailed uint64 `json:"-"`
+	ResumeBytesDone     uint64 `json:"-"`
+	ResumeBytesFailed   uint64 `json:"-"`
+
+	// Filled on startup/restarts.
+	QueuedBuckets []string
+
+	// Filled during heal.
+	HealedBuckets []string
+	// Add future tracking capabilities
+	// Be sure that they are included in toHealingDisk
+}
+
+// loadHealingTracker will load the healing tracker from the supplied disk.
+// The disk ID will be validated against the loaded one.
+func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker, error) {
+	if disk == nil {
+		return nil, errors.New("loadHealingTracker: nil disk given")
+	}
+	diskID, err := disk.GetDiskID()
+	if err != nil {
+		return nil, err
+	}
+	b, err := disk.ReadAll(ctx, minioMetaBucket,
+		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename))
+	if err != nil {
+		return nil, err
+	}
+	var h healingTracker
+	_, err = h.UnmarshalMsg(b)
+	if err != nil {
+		return nil, err
+	}
+	if h.ID != diskID && h.ID != "" {
+		return nil, fmt.Errorf("loadHealingTracker: disk id mismatch expected %s, got %s", h.ID, diskID)
+	}
+	h.disk = disk
+	h.ID = diskID
+	return &h, nil
+}
+
+// newHealingTracker will create a new healing tracker for the disk.
+func newHealingTracker(disk StorageAPI) *healingTracker {
+	diskID, _ := disk.GetDiskID()
+	h := healingTracker{
+		disk:     disk,
+		ID:       diskID,
+		Path:     disk.String(),
+		Endpoint: disk.Endpoint().String(),
+		Started:  time.Now().UTC(),
+	}
+	h.PoolIndex, h.SetIndex, h.DiskIndex = disk.GetDiskLoc()
+	return &h
+}
+
+// update will update the tracker on the disk.
+// If the tracker has been deleted an error is returned.
+func (h *healingTracker) update(ctx context.Context) error {
+	if h.disk.Healing() == nil {
+		return fmt.Errorf("healingTracker: disk %q is not marked as healing", h.ID)
+	}
+	if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
+		h.ID, _ = h.disk.GetDiskID()
+		h.PoolIndex, h.SetIndex, h.DiskIndex = h.disk.GetDiskLoc()
+	}
+	return h.save(ctx)
+}
+
+// save will unconditionally save the tracker and will be created if not existing.
+func (h *healingTracker) save(ctx context.Context) error {
+	if h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
+		// Attempt to get location.
+		if api := newObjectLayerFn(); api != nil {
+			if ep, ok := api.(*erasureServerPools); ok {
+				h.PoolIndex, h.SetIndex, h.DiskIndex, _ = ep.getPoolAndSet(h.ID)
+			}
+		}
+	}
+	h.LastUpdate = time.Now().UTC()
+	htrackerBytes, err := h.MarshalMsg(nil)
+	if err != nil {
+		return err
+	}
+	globalBackgroundHealState.updateHealStatus(h)
+	return h.disk.WriteAll(ctx, minioMetaBucket,
+		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
+		htrackerBytes)
+}
+
+// delete the tracker on disk.
+func (h *healingTracker) delete(ctx context.Context) error {
+	return h.disk.Delete(ctx, minioMetaBucket,
+		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
+		false)
+}
+
+func (h *healingTracker) isHealed(bucket string) bool {
+	for _, v := range h.HealedBuckets {
+		if v == bucket {
+			return true
+		}
+	}
+	return false
+}
+
+// resume will reset progress to the numbers at the start of the bucket.
+func (h *healingTracker) resume() {
+	h.ObjectsHealed = h.ResumeObjectsHealed
+	h.ObjectsFailed = h.ResumeObjectsFailed
+	h.BytesDone = h.ResumeBytesDone
+	h.BytesFailed = h.ResumeBytesFailed
+}
+
+// bucketDone should be called when a bucket is done healing.
+// Adds the bucket to the list of healed buckets and updates resume numbers.
+func (h *healingTracker) bucketDone(bucket string) {
+	h.ResumeObjectsHealed = h.ObjectsHealed
+	h.ResumeObjectsFailed = h.ObjectsFailed
+	h.ResumeBytesDone = h.BytesDone
+	h.ResumeBytesFailed = h.BytesFailed
+	h.HealedBuckets = append(h.HealedBuckets, bucket)
+	for i, b := range h.QueuedBuckets {
+		if b == bucket {
+			// Delete...
+			h.QueuedBuckets = append(h.QueuedBuckets[:i], h.QueuedBuckets[i+1:]...)
+		}
+	}
+}
+
+// setQueuedBuckets will add buckets, but exclude any that is already in h.HealedBuckets.
+// Order is preserved.
+func (h *healingTracker) setQueuedBuckets(buckets []BucketInfo) {
+	s := set.CreateStringSet(h.HealedBuckets...)
+	h.QueuedBuckets = make([]string, 0, len(buckets))
+	for _, b := range buckets {
+		if !s.Contains(b.Name) {
+			h.QueuedBuckets = append(h.QueuedBuckets, b.Name)
+		}
+	}
+}
+
+func (h *healingTracker) printTo(writer io.Writer) {
+	b, err := json.MarshalIndent(h, "", "  ")
+	if err != nil {
+		writer.Write([]byte(err.Error()))
+	}
+	writer.Write(b)
+}
+
+// toHealingDisk converts the information to madmin.HealingDisk
+func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
+	return madmin.HealingDisk{
+		ID:            h.ID,
+		Endpoint:      h.Endpoint,
+		PoolIndex:     h.PoolIndex,
+		SetIndex:      h.SetIndex,
+		DiskIndex:     h.DiskIndex,
+		Path:          h.Path,
+		Started:       h.Started.UTC(),
+		LastUpdate:    h.LastUpdate.UTC(),
+		ObjectsHealed: h.ObjectsHealed,
+		ObjectsFailed: h.ObjectsFailed,
+		BytesDone:     h.BytesDone,
+		BytesFailed:   h.BytesFailed,
+		Bucket:        h.Bucket,
+		Object:        h.Object,
+		QueuedBuckets: h.QueuedBuckets,
+		HealedBuckets: h.HealedBuckets,
+	}
 }
 
 func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
@@ -89,7 +287,7 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 			disk, _, err := connectEndpoint(endpoint)
 			if errors.Is(err, errUnformattedDisk) {
 				disksToHeal = append(disksToHeal, endpoint)
-			} else if err == nil && disk != nil && disk.Healing() {
+			} else if err == nil && disk != nil && disk.Healing() != nil {
 				disksToHeal = append(disksToHeal, disk.Endpoint())
 			}
 		}
@@ -113,7 +311,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 	// Perform automatic disk healing when a disk is replaced locally.
 	diskCheckTimer := time.NewTimer(defaultMonitorNewDiskInterval)
 	defer diskCheckTimer.Stop()
-wait:
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -122,9 +320,9 @@ wait:
 			// Reset to next interval.
 			diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
 
-			var erasureSetInZoneDisksToHeal []map[int][]StorageAPI
+			var erasureSetInPoolDisksToHeal []map[int][]StorageAPI
 
-			healDisks := globalBackgroundHealState.getHealLocalDisks()
+			healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints()
 			if len(healDisks) > 0 {
 				// Reformat disks
 				bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
@@ -135,14 +333,14 @@ wait:
 				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
 					len(healDisks)))
 
-				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
+				erasureSetInPoolDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
 				for i := range z.serverPools {
-					erasureSetInZoneDisksToHeal[i] = map[int][]StorageAPI{}
+					erasureSetInPoolDisksToHeal[i] = map[int][]StorageAPI{}
 				}
 			}
 
 			if serverDebugLog {
-				console.Debugf("disk check timer fired, attempting to heal %d drives\n", len(healDisks))
+				console.Debugf(color.Green("healDisk:")+" disk check timer fired, attempting to heal %d drives\n", len(healDisks))
 			}
 
 			// heal only if new disks found.
@@ -153,7 +351,7 @@ wait:
 					continue
 				}
 
-				poolIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
+				poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
 				if poolIdx < 0 {
 					continue
 				}
@@ -168,60 +366,81 @@ wait:
 					continue
 				}
 
-				erasureSetInZoneDisksToHeal[poolIdx][setIndex] = append(erasureSetInZoneDisksToHeal[poolIdx][setIndex], disk)
+				erasureSetInPoolDisksToHeal[poolIdx][setIndex] = append(erasureSetInPoolDisksToHeal[poolIdx][setIndex], disk)
 			}
 
 			buckets, _ := z.ListBuckets(ctx)
 
+			buckets = append(buckets, BucketInfo{
+				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
+			})
+
+			// Buckets data are dispersed in multiple zones/sets, make
+			// sure to heal all bucket metadata configuration.
+			buckets = append(buckets, []BucketInfo{
+				{Name: pathJoin(minioMetaBucket, bucketMetaPrefix)},
+			}...)
+
 			// Heal latest buckets first.
 			sort.Slice(buckets, func(i, j int) bool {
+				a, b := strings.HasPrefix(buckets[i].Name, minioMetaBucket), strings.HasPrefix(buckets[j].Name, minioMetaBucket)
+				if a != b {
+					return a
+				}
 				return buckets[i].Created.After(buckets[j].Created)
 			})
 
-			for i, setMap := range erasureSetInZoneDisksToHeal {
+			// TODO(klauspost): This will block until all heals are done,
+			// in the future this should be able to start healing other sets at once.
+			var wg sync.WaitGroup
+			for i, setMap := range erasureSetInPoolDisksToHeal {
+				i := i
 				for setIndex, disks := range setMap {
-					for _, disk := range disks {
-						logger.Info("Healing disk '%s' on %s pool", disk, humanize.Ordinal(i+1))
+					if len(disks) == 0 {
+						continue
+					}
+					wg.Add(1)
+					go func(setIndex int, disks []StorageAPI) {
+						defer wg.Done()
+						for _, disk := range disks {
+							logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(i+1))
 
-						// So someone changed the drives underneath, healing tracker missing.
-						if !disk.Healing() {
-							logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool", disk, humanize.Ordinal(i+1))
-							diskID, err := disk.GetDiskID()
+							// So someone changed the drives underneath, healing tracker missing.
+							tracker, err := loadHealingTracker(ctx, disk)
 							if err != nil {
-								logger.LogIf(ctx, err)
-								// reading format.json failed or not found, proceed to look
-								// for new disks to be healed again, we cannot proceed further.
-								goto wait
+								logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool", disk, humanize.Ordinal(i+1))
+								tracker = newHealingTracker(disk)
 							}
 
-							if err := saveHealingTracker(disk, diskID); err != nil {
+							tracker.PoolIndex, tracker.SetIndex, tracker.DiskIndex = disk.GetDiskLoc()
+							tracker.setQueuedBuckets(buckets)
+							if err := tracker.save(ctx); err != nil {
 								logger.LogIf(ctx, err)
 								// Unable to write healing tracker, permission denied or some
 								// other unexpected error occurred. Proceed to look for new
 								// disks to be healed again, we cannot proceed further.
-								goto wait
+								return
 							}
+
+							err = z.serverPools[i].sets[setIndex].healErasureSet(ctx, buckets, tracker)
+							if err != nil {
+								logger.LogIf(ctx, err)
+								continue
+							}
+
+							logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))
+							var buf bytes.Buffer
+							tracker.printTo(&buf)
+							logger.Info("Summary:\n%s", buf.String())
+							logger.LogIf(ctx, tracker.delete(ctx))
+
+							// Only upon success pop the healed disk.
+							globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())
 						}
-
-						lbDisks := z.serverPools[i].sets[setIndex].getOnlineDisks()
-						if err := healErasureSet(ctx, setIndex, buckets, lbDisks); err != nil {
-							logger.LogIf(ctx, err)
-							continue
-						}
-
-						logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))
-
-						if err := disk.Delete(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
-							healingTrackerFilename, false); err != nil && !errors.Is(err, errFileNotFound) {
-							logger.LogIf(ctx, err)
-							continue
-						}
-
-						// Only upon success pop the healed disk.
-						globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())
-					}
+					}(setIndex, disks)
 				}
 			}
+			wg.Wait()
 		}
 	}
 }

cmd/background-newdisks-heal-ops_gen.go

@@ -30,6 +30,146 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ID")
 				return
 			}
+		case "PoolIndex":
+			z.PoolIndex, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "PoolIndex")
+				return
+			}
+		case "SetIndex":
+			z.SetIndex, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "SetIndex")
+				return
+			}
+		case "DiskIndex":
+			z.DiskIndex, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "DiskIndex")
+				return
+			}
+		case "Path":
+			z.Path, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Path")
+				return
+			}
+		case "Endpoint":
+			z.Endpoint, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "Started":
+			z.Started, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "Started")
+				return
+			}
+		case "LastUpdate":
+			z.LastUpdate, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "LastUpdate")
+				return
+			}
+		case "ObjectsHealed":
+			z.ObjectsHealed, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ObjectsHealed")
+				return
+			}
+		case "ObjectsFailed":
+			z.ObjectsFailed, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ObjectsFailed")
+				return
+			}
+		case "BytesDone":
+			z.BytesDone, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "BytesDone")
+				return
+			}
+		case "BytesFailed":
+			z.BytesFailed, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "BytesFailed")
+				return
+			}
+		case "Bucket":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Object":
+			z.Object, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		case "ResumeObjectsHealed":
+			z.ResumeObjectsHealed, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeObjectsHealed")
+				return
+			}
+		case "ResumeObjectsFailed":
+			z.ResumeObjectsFailed, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeObjectsFailed")
+				return
+			}
+		case "ResumeBytesDone":
+			z.ResumeBytesDone, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeBytesDone")
+				return
+			}
+		case "ResumeBytesFailed":
+			z.ResumeBytesFailed, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeBytesFailed")
+				return
+			}
+		case "QueuedBuckets":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "QueuedBuckets")
+				return
+			}
+			if cap(z.QueuedBuckets) >= int(zb0002) {
+				z.QueuedBuckets = (z.QueuedBuckets)[:zb0002]
+			} else {
+				z.QueuedBuckets = make([]string, zb0002)
+			}
+			for za0001 := range z.QueuedBuckets {
+				z.QueuedBuckets[za0001], err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "QueuedBuckets", za0001)
+					return
+				}
+			}
+		case "HealedBuckets":
+			var zb0003 uint32
+			zb0003, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "HealedBuckets")
+				return
+			}
+			if cap(z.HealedBuckets) >= int(zb0003) {
+				z.HealedBuckets = (z.HealedBuckets)[:zb0003]
+			} else {
+				z.HealedBuckets = make([]string, zb0003)
+			}
+			for za0002 := range z.HealedBuckets {
+				z.HealedBuckets[za0002], err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "HealedBuckets", za0002)
+					return
+				}
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -42,10 +182,10 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 }
 
 // EncodeMsg implements msgp.Encodable
-func (z healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 1
+func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 20
 	// write "ID"
-	err = en.Append(0x81, 0xa2, 0x49, 0x44)
+	err = en.Append(0xde, 0x0, 0x14, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -54,16 +194,283 @@ func (z healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ID")
 		return
 	}
+	// write "PoolIndex"
+	err = en.Append(0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.PoolIndex)
+	if err != nil {
+		err = msgp.WrapError(err, "PoolIndex")
+		return
+	}
+	// write "SetIndex"
+	err = en.Append(0xa8, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.SetIndex)
+	if err != nil {
+		err = msgp.WrapError(err, "SetIndex")
+		return
+	}
+	// write "DiskIndex"
+	err = en.Append(0xa9, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.DiskIndex)
+	if err != nil {
+		err = msgp.WrapError(err, "DiskIndex")
+		return
+	}
+	// write "Path"
+	err = en.Append(0xa4, 0x50, 0x61, 0x74, 0x68)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Path)
+	if err != nil {
+		err = msgp.WrapError(err, "Path")
+		return
+	}
+	// write "Endpoint"
+	err = en.Append(0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Endpoint)
+	if err != nil {
+		err = msgp.WrapError(err, "Endpoint")
+		return
+	}
+	// write "Started"
+	err = en.Append(0xa7, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.Started)
+	if err != nil {
+		err = msgp.WrapError(err, "Started")
+		return
+	}
+	// write "LastUpdate"
+	err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.LastUpdate)
+	if err != nil {
+		err = msgp.WrapError(err, "LastUpdate")
+		return
+	}
+	// write "ObjectsHealed"
+	err = en.Append(0xad, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ObjectsHealed)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjectsHealed")
+		return
+	}
+	// write "ObjectsFailed"
+	err = en.Append(0xad, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ObjectsFailed)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjectsFailed")
+		return
+	}
+	// write "BytesDone"
+	err = en.Append(0xa9, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.BytesDone)
+	if err != nil {
+		err = msgp.WrapError(err, "BytesDone")
+		return
+	}
+	// write "BytesFailed"
+	err = en.Append(0xab, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.BytesFailed)
+	if err != nil {
+		err = msgp.WrapError(err, "BytesFailed")
+		return
+	}
+	// write "Bucket"
+	err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "Object"
+	err = en.Append(0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Object)
+	if err != nil {
+		err = msgp.WrapError(err, "Object")
+		return
+	}
+	// write "ResumeObjectsHealed"
+	err = en.Append(0xb3, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ResumeObjectsHealed)
+	if err != nil {
+		err = msgp.WrapError(err, "ResumeObjectsHealed")
+		return
+	}
+	// write "ResumeObjectsFailed"
+	err = en.Append(0xb3, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ResumeObjectsFailed)
+	if err != nil {
+		err = msgp.WrapError(err, "ResumeObjectsFailed")
+		return
+	}
+	// write "ResumeBytesDone"
+	err = en.Append(0xaf, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ResumeBytesDone)
+	if err != nil {
+		err = msgp.WrapError(err, "ResumeBytesDone")
+		return
+	}
+	// write "ResumeBytesFailed"
+	err = en.Append(0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ResumeBytesFailed)
+	if err != nil {
+		err = msgp.WrapError(err, "ResumeBytesFailed")
+		return
+	}
+	// write "QueuedBuckets"
+	err = en.Append(0xad, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.QueuedBuckets)))
+	if err != nil {
+		err = msgp.WrapError(err, "QueuedBuckets")
+		return
+	}
+	for za0001 := range z.QueuedBuckets {
+		err = en.WriteString(z.QueuedBuckets[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "QueuedBuckets", za0001)
+			return
+		}
+	}
+	// write "HealedBuckets"
+	err = en.Append(0xad, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.HealedBuckets)))
+	if err != nil {
+		err = msgp.WrapError(err, "HealedBuckets")
+		return
+	}
+	for za0002 := range z.HealedBuckets {
+		err = en.WriteString(z.HealedBuckets[za0002])
+		if err != nil {
+			err = msgp.WrapError(err, "HealedBuckets", za0002)
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
-func (z healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
+func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 1
+	// map header, size 20
 	// string "ID"
-	o = append(o, 0x81, 0xa2, 0x49, 0x44)
+	o = append(o, 0xde, 0x0, 0x14, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
+	// string "PoolIndex"
+	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	o = msgp.AppendInt(o, z.PoolIndex)
+	// string "SetIndex"
+	o = append(o, 0xa8, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	o = msgp.AppendInt(o, z.SetIndex)
+	// string "DiskIndex"
+	o = append(o, 0xa9, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	o = msgp.AppendInt(o, z.DiskIndex)
+	// string "Path"
+	o = append(o, 0xa4, 0x50, 0x61, 0x74, 0x68)
+	o = msgp.AppendString(o, z.Path)
+	// string "Endpoint"
+	o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	o = msgp.AppendString(o, z.Endpoint)
+	// string "Started"
+	o = append(o, 0xa7, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64)
+	o = msgp.AppendTime(o, z.Started)
+	// string "LastUpdate"
+	o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
+	o = msgp.AppendTime(o, z.LastUpdate)
+	// string "ObjectsHealed"
+	o = append(o, 0xad, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ObjectsHealed)
+	// string "ObjectsFailed"
+	o = append(o, 0xad, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ObjectsFailed)
+	// string "BytesDone"
+	o = append(o, 0xa9, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65)
+	o = msgp.AppendUint64(o, z.BytesDone)
+	// string "BytesFailed"
+	o = append(o, 0xab, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.BytesFailed)
+	// string "Bucket"
+	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "Object"
+	o = append(o, 0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74)
+	o = msgp.AppendString(o, z.Object)
+	// string "ResumeObjectsHealed"
+	o = append(o, 0xb3, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ResumeObjectsHealed)
+	// string "ResumeObjectsFailed"
+	o = append(o, 0xb3, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ResumeObjectsFailed)
+	// string "ResumeBytesDone"
+	o = append(o, 0xaf, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65)
+	o = msgp.AppendUint64(o, z.ResumeBytesDone)
+	// string "ResumeBytesFailed"
+	o = append(o, 0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ResumeBytesFailed)
+	// string "QueuedBuckets"
+	o = append(o, 0xad, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.QueuedBuckets)))
+	for za0001 := range z.QueuedBuckets {
+		o = msgp.AppendString(o, z.QueuedBuckets[za0001])
+	}
+	// string "HealedBuckets"
+	o = append(o, 0xad, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.HealedBuckets)))
+	for za0002 := range z.HealedBuckets {
+		o = msgp.AppendString(o, z.HealedBuckets[za0002])
+	}
 	return
 }
 
@@ -91,6 +498,146 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ID")
 				return
 			}
+		case "PoolIndex":
+			z.PoolIndex, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PoolIndex")
+				return
+			}
+		case "SetIndex":
+			z.SetIndex, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "SetIndex")
+				return
+			}
+		case "DiskIndex":
+			z.DiskIndex, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "DiskIndex")
+				return
+			}
+		case "Path":
+			z.Path, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Path")
+				return
+			}
+		case "Endpoint":
+			z.Endpoint, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "Started":
+			z.Started, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Started")
+				return
+			}
+		case "LastUpdate":
+			z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LastUpdate")
+				return
+			}
+		case "ObjectsHealed":
+			z.ObjectsHealed, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjectsHealed")
+				return
+			}
+		case "ObjectsFailed":
+			z.ObjectsFailed, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjectsFailed")
+				return
+			}
+		case "BytesDone":
+			z.BytesDone, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BytesDone")
+				return
+			}
+		case "BytesFailed":
+			z.BytesFailed, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BytesFailed")
+				return
+			}
+		case "Bucket":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Object":
+			z.Object, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		case "ResumeObjectsHealed":
+			z.ResumeObjectsHealed, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeObjectsHealed")
+				return
+			}
+		case "ResumeObjectsFailed":
+			z.ResumeObjectsFailed, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeObjectsFailed")
+				return
+			}
+		case "ResumeBytesDone":
+			z.ResumeBytesDone, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeBytesDone")
+				return
+			}
+		case "ResumeBytesFailed":
+			z.ResumeBytesFailed, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeBytesFailed")
+				return
+			}
+		case "QueuedBuckets":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "QueuedBuckets")
+				return
+			}
+			if cap(z.QueuedBuckets) >= int(zb0002) {
+				z.QueuedBuckets = (z.QueuedBuckets)[:zb0002]
+			} else {
+				z.QueuedBuckets = make([]string, zb0002)
+			}
+			for za0001 := range z.QueuedBuckets {
+				z.QueuedBuckets[za0001], bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "QueuedBuckets", za0001)
+					return
+				}
+			}
+		case "HealedBuckets":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "HealedBuckets")
+				return
+			}
+			if cap(z.HealedBuckets) >= int(zb0003) {
+				z.HealedBuckets = (z.HealedBuckets)[:zb0003]
+			} else {
+				z.HealedBuckets = make([]string, zb0003)
+			}
+			for za0002 := range z.HealedBuckets {
+				z.HealedBuckets[za0002], bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "HealedBuckets", za0002)
+					return
+				}
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -104,7 +651,14 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z healingTracker) Msgsize() (s int) {
-	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID)
+func (z *healingTracker) Msgsize() (s int) {
+	s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 10 + msgp.IntSize + 9 + msgp.IntSize + 10 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Path) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 8 + msgp.TimeSize + 11 + msgp.TimeSize + 14 + msgp.Uint64Size + 14 + msgp.Uint64Size + 10 + msgp.Uint64Size + 12 + msgp.Uint64Size + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 20 + msgp.Uint64Size + 20 + msgp.Uint64Size + 16 + msgp.Uint64Size + 18 + msgp.Uint64Size + 14 + msgp.ArrayHeaderSize
+	for za0001 := range z.QueuedBuckets {
+		s += msgp.StringPrefixSize + len(z.QueuedBuckets[za0001])
+	}
+	s += 14 + msgp.ArrayHeaderSize
+	for za0002 := range z.HealedBuckets {
+		s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002])
+	}
 	return
 }

cmd/benchmark-utils_test.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"io/ioutil"
 	"math"
 	"math/rand"
 	"strconv"
@@ -175,56 +174,6 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
 	runPutObjectBenchmarkParallel(b, objLayer, objSize)
 }
 
-// Benchmark utility functions for ObjectLayer.GetObject().
-// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
-func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
-	// obtains random bucket name.
-	bucket := getRandomBucketName()
-	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	textData := generateBytesData(objSize)
-
-	// generate etag for the generated data.
-	// etag of the data to written is required as input for PutObject.
-	// PutObject is the functions which writes the data onto the FS/Erasure backend.
-
-	// get text data generated for number of bytes equal to object size.
-	md5hex := getMD5Hash(textData)
-	sha256hex := ""
-
-	for i := 0; i < 10; i++ {
-		// insert the object.
-		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
-		if err != nil {
-			b.Fatal(err)
-		}
-		if objInfo.ETag != md5hex {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
-		}
-	}
-
-	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
-	b.ReportAllocs()
-	// the actual benchmark for GetObject starts here. Reset the benchmark timer.
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		var buffer = new(bytes.Buffer)
-		err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "", ObjectOptions{})
-		if err != nil {
-			b.Error(err)
-		}
-	}
-	// Benchmark ends here. Stop timer.
-	b.StopTimer()
-
-}
-
 // randomly picks a character and returns its equivalent byte array.
 func getRandomByte() []byte {
 	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
@@ -240,38 +189,6 @@ func generateBytesData(size int) []byte {
 	return bytes.Repeat(getRandomByte(), size)
 }
 
-// creates Erasure/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
-func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
-	// create a temp Erasure/FS backend.
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
-	if err != nil {
-		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
-	}
-	// cleaning up the backend by removing all the directories and files created.
-	defer removeRoots(disks)
-
-	//  uses *testing.B and the object Layer to run the benchmark.
-	runGetObjectBenchmark(b, objLayer, objSize)
-}
-
-// creates Erasure/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
-func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) {
-	// create a temp Erasure/FS backend.
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
-	if err != nil {
-		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
-	}
-	// cleaning up the backend by removing all the directories and files created.
-	defer removeRoots(disks)
-
-	//  uses *testing.B and the object Layer to run the benchmark.
-	runGetObjectBenchmarkParallel(b, objLayer, objSize)
-}
-
 // Parallel benchmark utility functions for ObjectLayer.PutObject().
 // Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
 func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
@@ -315,58 +232,3 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// Benchmark ends here. Stop timer.
 	b.StopTimer()
 }
-
-// Parallel benchmark utility functions for ObjectLayer.GetObject().
-// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
-func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
-	// obtains random bucket name.
-	bucket := getRandomBucketName()
-	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// get text data generated for number of bytes equal to object size.
-	textData := generateBytesData(objSize)
-	// generate md5sum for the generated data.
-	// md5sum of the data to written is required as input for PutObject.
-	// PutObject is the functions which writes the data onto the FS/Erasure backend.
-
-	md5hex := getMD5Hash([]byte(textData))
-	sha256hex := ""
-
-	for i := 0; i < 10; i++ {
-		// insert the object.
-		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
-		if err != nil {
-			b.Fatal(err)
-		}
-		if objInfo.ETag != md5hex {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
-		}
-	}
-
-	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
-	b.ReportAllocs()
-	// the actual benchmark for GetObject starts here. Reset the benchmark timer.
-	b.ResetTimer()
-	b.RunParallel(func(pb *testing.PB) {
-		i := 0
-		for pb.Next() {
-			err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "", ObjectOptions{})
-			if err != nil {
-				b.Error(err)
-			}
-			i++
-			if i == 10 {
-				i = 0
-			}
-		}
-	})
-	// Benchmark ends here. Stop timer.
-	b.StopTimer()
-
-}

cmd/bitrot-streaming.go

@@ -149,10 +149,9 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err := &errHashMismatch{fmt.Sprintf("Disk: %s  -> %s/%s - content hash does not match - expected %s, got %s",
-			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
-		logger.LogIf(GlobalContext, err)
-		return 0, err
+		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s  -> %s/%s - content hash does not match - expected %s, got %s",
+			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))))
+		return 0, errFileCorrupt
 	}
 	b.currOffset += int64(len(buf))
 	return len(buf), nil

cmd/bitrot.go

@@ -17,13 +17,13 @@
 package cmd
 
 import (
+	"crypto/sha256"
 	"errors"
 	"hash"
 	"io"
 
 	"github.com/minio/highwayhash"
 	"github.com/minio/minio/cmd/logger"
-	sha256 "github.com/minio/sha256-simd"
 	"golang.org/x/crypto/blake2b"
 )
 

cmd/bitrot_test.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"testing"
 )
@@ -28,7 +27,7 @@ import (
 func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	tmpDir, err := ioutil.TempDir("", "")
 	if err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmpDir)
 
@@ -46,35 +45,35 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 
 	_, err = writer.Write([]byte("aaaaaaaaaa"))
 	if err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 	_, err = writer.Write([]byte("aaaaaaaaaa"))
 	if err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 	_, err = writer.Write([]byte("aaaaaaaaaa"))
 	if err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 	_, err = writer.Write([]byte("aaaaa"))
 	if err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 	writer.(io.Closer).Close()
 
 	reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
 	b := make([]byte, 10)
 	if _, err = reader.ReadAt(b, 0); err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 	if _, err = reader.ReadAt(b, 10); err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 	if _, err = reader.ReadAt(b, 20); err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 	if _, err = reader.ReadAt(b[:5], 30); err != nil {
-		log.Fatal(err)
+		t.Fatal(err)
 	}
 }
 

cmd/bucket-encryption-handlers.go

@@ -37,7 +37,7 @@ const (
 func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketEncryption")
 
-	defer logger.AuditLog(w, r, "PutBucketEncryption", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -102,7 +102,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketEncryption")
 
-	defer logger.AuditLog(w, r, "GetBucketEncryption", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketEncryption")
 
-	defer logger.AuditLog(w, r, "DeleteBucketEncryption", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-handlers.go

@@ -17,16 +17,19 @@
 package cmd
 
 import (
+	"crypto/subtle"
 	"encoding/base64"
 	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
+	"net/textproto"
 	"net/url"
 	"path"
-	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
@@ -72,7 +75,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 
 	// Get buckets in the DNS
 	dnsBuckets, err := globalDNSConfig.List()
-	if err != nil && err != dns.ErrNoEntriesFound && err != dns.ErrNotImplemented {
+	if err != nil && !IsErrIgnored(err, dns.ErrNoEntriesFound, dns.ErrNotImplemented, dns.ErrDomainMissing) {
 		logger.LogIf(GlobalContext, err)
 		return
 	}
@@ -80,6 +83,10 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	bucketsSet := set.NewStringSet()
 	bucketsToBeUpdated := set.NewStringSet()
 	bucketsInConflict := set.NewStringSet()
+
+	// This means that domain is updated, we should update
+	// all bucket entries with new domain name.
+	domainMissing := err == dns.ErrDomainMissing
 	if dnsBuckets != nil {
 		for _, bucket := range buckets {
 			bucketsSet.Add(bucket.Name)
@@ -89,11 +96,16 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 				continue
 			}
 			if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() && !domainMissing {
 					// No difference in terms of domainIPs and nothing
 					// has changed so we don't change anything on the etcd.
+					//
+					// Additionally also check if domain is updated/missing with more
+					// entries, if that is the case we should update the
+					// new domain entries as well.
 					continue
 				}
+
 				// if domain IPs intersect then it won't be an empty set.
 				// such an intersection means that bucket exists on etcd.
 				// but if we do see a difference with local domain IPs with
@@ -102,6 +114,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 				bucketsToBeUpdated.Add(bucket.Name)
 				continue
 			}
+
 			// No IPs seem to intersect, this means that bucket exists but has
 			// different IP addresses perhaps from a different deployment.
 			// bucket names are globally unique in federation at a given
@@ -112,8 +125,11 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	}
 
 	// Add/update buckets that are not registered with the DNS
-	g := errgroup.WithNErrs(len(buckets))
 	bucketsToBeUpdatedSlice := bucketsToBeUpdated.ToSlice()
+	g := errgroup.WithNErrs(len(bucketsToBeUpdatedSlice)).WithConcurrency(50)
+	ctx, cancel := g.WithCancelOnError(GlobalContext)
+	defer cancel()
+
 	for index := range bucketsToBeUpdatedSlice {
 		index := index
 		g.Go(func() error {
@@ -121,16 +137,16 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 		}, index)
 	}
 
-	for _, err := range g.Wait() {
-		if err != nil {
-			logger.LogIf(GlobalContext, err)
-		}
+	if err := g.WaitErr(); err != nil {
+		logger.LogIf(ctx, err)
+		return
 	}
 
 	for _, bucket := range bucketsInConflict.ToSlice() {
-		logger.LogIf(GlobalContext, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", bucket, globalDomainIPs.ToSlice()))
+		logger.LogIf(ctx, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket by a different tenant. This local bucket will be ignored. Bucket names are globally unique in federated deployments. Use path style requests on following addresses '%v' to access this bucket", bucket, globalDomainIPs.ToSlice()))
 	}
 
+	var wg sync.WaitGroup
 	// Remove buckets that are in DNS for this server, but aren't local
 	for bucket, records := range dnsBuckets {
 		if bucketsSet.Contains(bucket) {
@@ -142,13 +158,18 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 			continue
 		}
 
-		// We go to here, so we know the bucket no longer exists,
-		// but is registered in DNS to this server
-		if err = globalDNSConfig.Delete(bucket); err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w",
-				bucket, err))
-		}
+		wg.Add(1)
+		go func(bucket string) {
+			defer wg.Done()
+			// We go to here, so we know the bucket no longer exists,
+			// but is registered in DNS to this server
+			if err := globalDNSConfig.Delete(bucket); err != nil {
+				logger.LogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w",
+					bucket, err))
+			}
+		}(bucket)
 	}
+	wg.Wait()
 }
 
 // GetBucketLocationHandler - GET Bucket location.
@@ -157,7 +178,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLocation")
 
-	defer logger.AuditLog(w, r, "GetBucketLocation", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -205,7 +226,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListMultipartUploads")
 
-	defer logger.AuditLog(w, r, "ListMultipartUploads", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -260,7 +281,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBuckets")
 
-	defer logger.AuditLog(w, r, "ListBuckets", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -280,7 +301,9 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	var bucketsInfo []BucketInfo
 	if globalDNSConfig != nil && globalBucketFederation {
 		dnsBuckets, err := globalDNSConfig.List()
-		if err != nil && err != dns.ErrNoEntriesFound {
+		if err != nil && !IsErrIgnored(err,
+			dns.ErrNoEntriesFound,
+			dns.ErrDomainMissing) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
@@ -290,6 +313,11 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 				Created: dnsRecords[0].CreationDate,
 			})
 		}
+
+		sort.Slice(bucketsInfo, func(i, j int) bool {
+			return bucketsInfo[i].Name < bucketsInfo[j].Name
+		})
+
 	} else {
 		// Invoke the list buckets.
 		var err error
@@ -309,7 +337,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 		// err will be nil here as we already called this function
 		// earlier in this request.
-		claims, _ := getClaimsFromToken(r, getSessionToken(r))
+		claims, _ := getClaimsFromToken(getSessionToken(r))
 		n := 0
 		// Use the following trick to filter in place
 		// https://github.com/golang/go/wiki/SliceTricks#filter-in-place
@@ -347,7 +375,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteMultipleObjects")
 
-	defer logger.AuditLog(w, r, "DeleteMultipleObjects", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -383,6 +411,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Convert object name delete objects if it has `/` in the beginning.
+	for i := range deleteObjects.Objects {
+		deleteObjects.Objects[i].ObjectName = trimLeadingSlash(deleteObjects.Objects[i].ObjectName)
+	}
+
 	// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
 	// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
 	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
@@ -399,6 +432,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		deleteObjectsFn = api.CacheAPI().DeleteObjects
 	}
 
+	// Return Malformed XML as S3 spec if the list of objects is empty
+	if len(deleteObjects.Objects) == 0 {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	var objectsToDelete = map[ObjectToDelete]int{}
 	getObjectInfoFn := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
@@ -519,7 +558,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		dindex := objectsToDelete[ObjectToDelete{
 			ObjectName:                    dObjects[i].ObjectName,
 			VersionID:                     dObjects[i].VersionID,
-			DeleteMarkerVersionID:         dObjects[i].DeleteMarkerVersionID,
 			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus,
 			DeleteMarkerReplicationStatus: dObjects[i].DeleteMarkerReplicationStatus,
 			PurgeTransitioned:             dObjects[i].PurgeTransitioned,
@@ -555,6 +593,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 	for _, dobj := range deletedObjects {
+		if dobj.ObjectName == "" {
+			continue
+		}
+
 		if replicateDeletes {
 			if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
 				dv := DeletedObjectVersionInfo{
@@ -566,22 +608,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 
 		if hasLifecycleConfig && dobj.PurgeTransitioned == lifecycle.TransitionComplete { // clean up transitioned tier
-			action := lifecycle.DeleteAction
-			if dobj.VersionID != "" {
-				action = lifecycle.DeleteVersionAction
-			}
-			deleteTransitionedObject(ctx, newObjectLayerFn(), bucket, dobj.ObjectName, lifecycle.ObjectOpts{
+			deleteTransitionedObject(ctx, objectAPI, bucket, dobj.ObjectName, lifecycle.ObjectOpts{
 				Name:         dobj.ObjectName,
 				VersionID:    dobj.VersionID,
 				DeleteMarker: dobj.DeleteMarker,
-			}, action, true)
+			}, false, true)
 		}
 
-	}
-	// Notify deleted event for objects.
-	for _, dobj := range deletedObjects {
 		eventName := event.ObjectRemovedDelete
-
 		objInfo := ObjectInfo{
 			Name:      dobj.ObjectName,
 			VersionID: dobj.VersionID,
@@ -611,7 +645,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucket")
 
-	defer logger.AuditLog(w, r, "PutBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -742,7 +776,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PostPolicyBucket")
 
-	defer logger.AuditLog(w, r, "PostPolicyBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -768,13 +802,15 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
 	if err != nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	// Make sure that the URL  does not contain object name.
-	if bucket != filepath.Clean(resource[1:]) {
+
+	// Make sure that the URL does not contain object name.
+	if bucket != path.Clean(resource[1:]) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -817,13 +853,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	defer fileBody.Close()
 
 	formValues.Set("Bucket", bucket)
-
 	if fileName != "" && strings.Contains(formValues.Get("Key"), "${filename}") {
 		// S3 feature to replace ${filename} found in Key form field
 		// by the filename attribute passed in multipart
 		formValues.Set("Key", strings.Replace(formValues.Get("Key"), "${filename}", fileName, -1))
 	}
-	object := formValues.Get("Key")
+	object := trimLeadingSlash(formValues.Get("Key"))
 
 	successRedirect := formValues.Get("success_action_redirect")
 	successStatus := formValues.Get("success_action_status")
@@ -837,12 +872,51 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	// Verify policy signature.
-	errCode := doesPolicySignatureMatch(formValues)
+	cred, errCode := doesPolicySignatureMatch(formValues)
 	if errCode != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
+	// Once signature is validated, check if the user has
+	// explicit permissions for the user.
+	{
+		token := formValues.Get(xhttp.AmzSecurityToken)
+		if token != "" && cred.AccessKey == "" {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoAccessKey), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if cred.IsServiceAccount() && token == "" {
+			token = cred.SessionToken
+		}
+
+		if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidToken), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		// Extract claims if any.
+		claims, err := getClaimsFromToken(token)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          iampolicy.PutObjectAction,
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			BucketName:      bucket,
+			ObjectName:      object,
+			IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
+			Claims:          claims,
+		}) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
 	if err != nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
@@ -882,20 +956,20 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Extract metadata to be saved from received Form.
 	metadata := make(map[string]string)
-	err = extractMetadataFromMap(ctx, formValues, metadata)
+	err = extractMetadataFromMime(ctx, textproto.MIMEHeader(formValues), metadata)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize, globalCLIContext.StrictS3Compat)
+	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	rawReader := hashReader
-	pReader := NewPutObjReader(rawReader, nil, nil)
+	pReader := NewPutObjReader(rawReader)
 	var objectEncryptionKey crypto.ObjectKey
 
 	// Check if bucket encryption is enabled
@@ -935,12 +1009,16 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			}
 			info := ObjectInfo{Size: fileSize}
 			// do not try to verify encrypted content
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize, globalCLIContext.StrictS3Compat)
+			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize)
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+				return
+			}
+			pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
-			pReader = NewPutObjReader(rawReader, hashReader, &objectEncryptionKey)
 		}
 	}
 
@@ -997,6 +1075,64 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 }
 
+// GetBucketPolicyStatusHandler -  Retrieves the policy status
+// for an MinIO bucket, indicating whether the bucket is public.
+func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketPolicyStatus")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrServerNotInitialized))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyStatusAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if anonymous (non-owner) has access to list objects.
+	readable := globalPolicySys.IsAllowed(policy.Args{
+		Action:          policy.ListBucketAction,
+		BucketName:      bucket,
+		ConditionValues: getConditionValues(r, "", "", nil),
+		IsOwner:         false,
+	})
+
+	// Check if anonymous (non-owner) has access to upload objects.
+	writable := globalPolicySys.IsAllowed(policy.Args{
+		Action:          policy.PutObjectAction,
+		BucketName:      bucket,
+		ConditionValues: getConditionValues(r, "", "", nil),
+		IsOwner:         false,
+	})
+
+	encodedSuccessResponse := encodeResponse(PolicyStatus{
+		IsPublic: func() string {
+			// Silly to have special 'boolean' values yes
+			// but complying with silly implementation
+			// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketPolicyStatus.html
+			if readable && writable {
+				return "TRUE"
+			}
+			return "FALSE"
+		}(),
+	})
+
+	writeSuccessResponseXML(w, encodedSuccessResponse)
+}
+
 // HeadBucketHandler - HEAD Bucket
 // ----------
 // This operation is useful to determine if a bucket exists.
@@ -1006,7 +1142,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadBucket")
 
-	defer logger.AuditLog(w, r, "HeadBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1036,7 +1172,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucket")
 
-	defer logger.AuditLog(w, r, "DeleteBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1124,7 +1260,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketObjectLockConfig")
 
-	defer logger.AuditLog(w, r, "PutBucketObjectLockConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1180,7 +1316,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketObjectLockConfig")
 
-	defer logger.AuditLog(w, r, "GetBucketObjectLockConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1218,7 +1354,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketTagging")
 
-	defer logger.AuditLog(w, r, "PutBucketTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1262,7 +1398,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketTagging")
 
-	defer logger.AuditLog(w, r, "GetBucketTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1300,7 +1436,7 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketTagging")
 
-	defer logger.AuditLog(w, r, "DeleteBucketTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1330,7 +1466,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 // Add a replication configuration on the specified bucket as specified in https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html
 func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketReplicationConfig")
-	defer logger.AuditLog(w, r, "PutBucketReplicationConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1394,7 +1530,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketReplicationConfig")
 
-	defer logger.AuditLog(w, r, "GetBucketReplicationConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1435,7 +1571,7 @@ func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWr
 // ----------
 func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketReplicationConfig")
-	defer logger.AuditLog(w, r, "DeleteBucketReplicationConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 

cmd/bucket-lifecycle-handlers.go

@@ -38,7 +38,7 @@ const (
 func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketLifecycle")
 
-	defer logger.AuditLog(w, r, "PutBucketLifecycle", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -103,7 +103,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLifecycle")
 
-	defer logger.AuditLog(w, r, "GetBucketLifecycle", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketLifecycle")
 
-	defer logger.AuditLog(w, r, "DeleteBucketLifecycle", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-lifecycle.go

@@ -65,6 +65,46 @@ func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
 
+type expiryTask struct {
+	objInfo       ObjectInfo
+	versionExpiry bool
+}
+
+type expiryState struct {
+	expiryCh chan expiryTask
+}
+
+func (es *expiryState) queueExpiryTask(oi ObjectInfo, rmVersion bool) {
+	select {
+	case es.expiryCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion}:
+	default:
+	}
+}
+
+var (
+	globalExpiryState *expiryState
+)
+
+func newExpiryState() *expiryState {
+	es := &expiryState{
+		expiryCh: make(chan expiryTask, 10000),
+	}
+	go func() {
+		<-GlobalContext.Done()
+		close(es.expiryCh)
+	}()
+	return es
+}
+
+func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
+	globalExpiryState = newExpiryState()
+	go func() {
+		for t := range globalExpiryState.expiryCh {
+			applyExpiryRule(ctx, objectAPI, t.objInfo, false, t.versionExpiry)
+		}
+	}()
+}
+
 type transitionState struct {
 	// add future metrics here
 	transitionCh chan ObjectInfo
@@ -205,16 +245,11 @@ func transitionSCInUse(ctx context.Context, lfc *lifecycle.Lifecycle, bucket, ar
 }
 
 // set PutObjectOptions for PUT operation to transition data to target cluster
-func putTransitionOpts(objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
+func putTransitionOpts(objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 
-	tag, err := tags.ParseObjectTags(objInfo.UserTags)
-	if err != nil {
-		return
-	}
 	putOpts = miniogo.PutObjectOptions{
 		UserMetadata:    meta,
-		UserTags:        tag.ToMap(),
 		ContentType:     objInfo.ContentType,
 		ContentEncoding: objInfo.ContentEncoding,
 		StorageClass:    objInfo.StorageClass,
@@ -224,29 +259,47 @@ func putTransitionOpts(objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
 			SourceETag:      objInfo.ETag,
 		},
 	}
-	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
+
+	if objInfo.UserTags != "" {
+		tag, _ := tags.ParseObjectTags(objInfo.UserTags)
+		if tag != nil {
+			putOpts.UserTags = tag.ToMap()
+		}
+	}
+
+	lkMap := caseInsensitiveMap(objInfo.UserDefined)
+	if lang, ok := lkMap.Lookup(xhttp.ContentLanguage); ok {
+		putOpts.ContentLanguage = lang
+	}
+	if disp, ok := lkMap.Lookup(xhttp.ContentDisposition); ok {
+		putOpts.ContentDisposition = disp
+	}
+	if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok {
+		putOpts.CacheControl = cc
+	}
+	if mode, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); ok {
 		rmode := miniogo.RetentionMode(mode)
 		putOpts.Mode = rmode
 	}
-	if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
+	if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
 		rdate, err := time.Parse(time.RFC3339, retainDateStr)
 		if err != nil {
-			return
+			return putOpts, err
 		}
 		putOpts.RetainUntilDate = rdate
 	}
-	if lhold, ok := objInfo.UserDefined[xhttp.AmzObjectLockLegalHold]; ok {
+	if lhold, ok := lkMap.Lookup(xhttp.AmzObjectLockLegalHold); ok {
 		putOpts.LegalHold = miniogo.LegalHoldStatus(lhold)
 	}
 
-	return
+	return putOpts, nil
 }
 
 // handle deletes of transitioned objects or object versions when one of the following is true:
 // 1. temporarily restored copies of objects (restored with the PostRestoreObject API) expired.
 // 2. life cycle expiry date is met on the object.
 // 3. Object is removed through DELETE api call
-func deleteTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket, object string, lcOpts lifecycle.ObjectOpts, action lifecycle.Action, isDeleteTierOnly bool) error {
+func deleteTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket, object string, lcOpts lifecycle.ObjectOpts, restoredObject, isDeleteTierOnly bool) error {
 	if lcOpts.TransitionStatus == "" && !isDeleteTierOnly {
 		return nil
 	}
@@ -266,46 +319,41 @@ func deleteTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket
 	var opts ObjectOptions
 	opts.Versioned = globalBucketVersioningSys.Enabled(bucket)
 	opts.VersionID = lcOpts.VersionID
-	switch action {
-	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+	if restoredObject {
 		// delete locally restored copy of object or object version
 		// from the source, while leaving metadata behind. The data on
 		// transitioned tier lies untouched and still accessible
 		opts.TransitionStatus = lcOpts.TransitionStatus
 		_, err = objectAPI.DeleteObject(ctx, bucket, object, opts)
 		return err
-	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
-		// When an object is past expiry, delete the data from transitioned tier and
-		// metadata from source
-		if err := tgt.RemoveObject(context.Background(), arn.Bucket, object, miniogo.RemoveObjectOptions{VersionID: lcOpts.VersionID}); err != nil {
-			logger.LogIf(ctx, err)
-		}
-
-		if isDeleteTierOnly {
-			return nil
-		}
-		_, err = objectAPI.DeleteObject(ctx, bucket, object, opts)
-		if err != nil {
-			return err
-		}
-		eventName := event.ObjectRemovedDelete
-		if lcOpts.DeleteMarker {
-			eventName = event.ObjectRemovedDeleteMarkerCreated
-		}
-		objInfo := ObjectInfo{
-			Name:         object,
-			VersionID:    lcOpts.VersionID,
-			DeleteMarker: lcOpts.DeleteMarker,
-		}
-		// Notify object deleted event.
-		sendEvent(eventArgs{
-			EventName:  eventName,
-			BucketName: bucket,
-			Object:     objInfo,
-			Host:       "Internal: [ILM-EXPIRY]",
-		})
 	}
 
+	// When an object is past expiry, delete the data from transitioned tier and
+	// metadata from source
+	if err := tgt.RemoveObject(context.Background(), arn.Bucket, object, miniogo.RemoveObjectOptions{VersionID: lcOpts.VersionID}); err != nil {
+		logger.LogIf(ctx, err)
+	}
+
+	if isDeleteTierOnly {
+		return nil
+	}
+
+	objInfo, err := objectAPI.DeleteObject(ctx, bucket, object, opts)
+	if err != nil {
+		return err
+	}
+	eventName := event.ObjectRemovedDelete
+	if lcOpts.DeleteMarker {
+		eventName = event.ObjectRemovedDeleteMarkerCreated
+	}
+	// Notify object deleted event.
+	sendEvent(eventArgs{
+		EventName:  eventName,
+		BucketName: bucket,
+		Object:     objInfo,
+		Host:       "Internal: [ILM-EXPIRY]",
+	})
+
 	// should never reach here
 	return nil
 }
@@ -345,7 +393,12 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo Object
 		return nil
 	}
 
-	putOpts := putTransitionOpts(oi)
+	putOpts, err := putTransitionOpts(oi)
+	if err != nil {
+		gr.Close()
+		return err
+
+	}
 	if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, putOpts); err != nil {
 		gr.Close()
 		return err
@@ -358,20 +411,19 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo Object
 	opts.TransitionStatus = lifecycle.TransitionComplete
 	eventName := event.ObjectTransitionComplete
 
-	_, err = objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
+	objInfo, err = objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
 	if err != nil {
 		eventName = event.ObjectTransitionFailed
 	}
+
 	// Notify object deleted event.
 	sendEvent(eventArgs{
 		EventName:  eventName,
-		BucketName: oi.Bucket,
-		Object: ObjectInfo{
-			Name:      oi.Name,
-			VersionID: opts.VersionID,
-		},
-		Host: "Internal: [ILM-Transition]",
+		BucketName: objInfo.Bucket,
+		Object:     objInfo,
+		Host:       "Internal: [ILM-Transition]",
 	})
+
 	return err
 }
 
@@ -640,11 +692,11 @@ func restoreTransitionedObject(ctx context.Context, bucket, object string, objAP
 		return err
 	}
 	defer gr.Close()
-	hashReader, err := hash.NewReader(gr, objInfo.Size, "", "", objInfo.Size, globalCLIContext.StrictS3Compat)
+	hashReader, err := hash.NewReader(gr, objInfo.Size, "", "", objInfo.Size)
 	if err != nil {
 		return err
 	}
-	pReader := NewPutObjReader(hashReader, nil, nil)
+	pReader := NewPutObjReader(hashReader)
 	opts := putRestoreOpts(bucket, object, rreq, objInfo)
 	opts.UserDefined[xhttp.AmzRestore] = fmt.Sprintf("ongoing-request=%t, expiry-date=%s", false, restoreExpiry.Format(http.TimeFormat))
 	if _, err := objAPI.PutObject(ctx, bucket, object, pReader, opts); err != nil {

cmd/bucket-listobjects-handlers.go

@@ -26,32 +26,25 @@ import (
 	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
-	inParallel := func(objects []ObjectInfo) {
-		g := errgroup.WithNErrs(len(objects))
-		for index := range objects {
-			index := index
-			g.Go(func() error {
-				objects[index].ETag = objects[index].GetActualETag(nil)
-				objects[index].Size, _ = objects[index].GetActualSize()
-				return nil
-			}, index)
-		}
-		g.Wait()
-	}
-	const maxConcurrent = 500
-	for {
-		if len(objects) < maxConcurrent {
-			inParallel(objects)
-			return
-		}
-		inParallel(objects[:maxConcurrent])
-		objects = objects[maxConcurrent:]
+	g := errgroup.WithNErrs(len(objects)).WithConcurrency(500)
+	_, cancel := g.WithCancelOnError(ctx)
+	defer cancel()
+	for index := range objects {
+		index := index
+		g.Go(func() error {
+			size, err := objects[index].GetActualSize()
+			if err == nil {
+				objects[index].Size = size
+			}
+			objects[index].ETag = objects[index].GetActualETag(nil)
+			return nil
+		}, index)
 	}
+	g.WaitErr()
 }
 
 // Validate all the ListObjects query arguments, returns an APIErrorCode
@@ -82,7 +75,7 @@ func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int
 func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectVersions")
 
-	defer logger.AuditLog(w, r, "ListObjectVersions", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -143,7 +136,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2M")
 
-	defer logger.AuditLog(w, r, "ListObjectsV2M", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -210,7 +203,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
 
-	defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -301,10 +294,6 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http
 	return proxyRequest(ctx, w, r, ep)
 }
 
-func proxyRequestByStringHash(ctx context.Context, w http.ResponseWriter, r *http.Request, str string) (success bool) {
-	return proxyRequestByNodeIndex(ctx, w, r, crcHashMod(str, len(globalProxyEndpoints)))
-}
-
 // ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 10000)
@@ -314,7 +303,7 @@ func proxyRequestByStringHash(ctx context.Context, w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV1")
 
-	defer logger.AuditLog(w, r, "ListObjectsV1", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -343,15 +332,6 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Forward the request using Source IP or bucket
-	forwardStr := handlers.GetSourceIPFromHeaders(r)
-	if forwardStr == "" {
-		forwardStr = bucket
-	}
-	if proxyRequestByStringHash(ctx, w, r, forwardStr) {
-		return
-	}
-
 	listObjects := objectAPI.ListObjects
 
 	// Inititate a list objects operation based on the input params.

cmd/bucket-notification-handlers.go

@@ -39,7 +39,7 @@ const (
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketNotification")
 
-	defer logger.AuditLog(w, r, "GetBucketNotification", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
@@ -111,7 +111,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketNotification")
 
-	defer logger.AuditLog(w, r, "PutBucketNotification", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {

cmd/bucket-policy-handlers.go

@@ -40,7 +40,7 @@ const (
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketPolicy")
 
-	defer logger.AuditLog(w, r, "PutBucketPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -106,7 +106,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketPolicy")
 
-	defer logger.AuditLog(w, r, "DeleteBucketPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -141,7 +141,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketPolicy")
 
-	defer logger.AuditLog(w, r, "GetBucketPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-replication.go

@@ -20,8 +20,9 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"runtime"
+	"reflect"
 	"strings"
+	"sync"
 	"time"
 
 	minio "github.com/minio/minio-go/v7"
@@ -129,19 +130,29 @@ func mustReplicater(ctx context.Context, bucket, object string, meta map[string]
 		opts.UserTags = tagStr
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
-	if tgt == nil || tgt.isOffline() {
-		return cfg.Replicate(opts), false
+	// the target online status should not be used here while deciding
+	// whether to replicate as the target could be temporarily down
+	if tgt != nil {
+		return cfg.Replicate(opts), tgt.replicateSync
 	}
-	return cfg.Replicate(opts), tgt.replicateSync
+	return cfg.Replicate(opts), false
 }
 
 // Standard headers that needs to be extracted from User metadata.
 var standardHeaders = []string{
-	"content-type",
-	"content-encoding",
+	xhttp.ContentType,
+	xhttp.CacheControl,
+	xhttp.ContentEncoding,
+	xhttp.ContentLanguage,
+	xhttp.ContentDisposition,
 	xhttp.AmzStorageClass,
 	xhttp.AmzObjectTagging,
 	xhttp.AmzBucketReplicationStatus,
+	xhttp.AmzObjectLockMode,
+	xhttp.AmzObjectLockRetainUntilDate,
+	xhttp.AmzObjectLockLegalHold,
+	xhttp.AmzTagCount,
+	xhttp.AmzServerSideEncryption,
 }
 
 // returns true if any of the objects being deleted qualifies for replication.
@@ -159,14 +170,8 @@ func hasReplicationRules(ctx context.Context, bucket string, objects []ObjectToD
 }
 
 // isStandardHeader returns true if header is a supported header and not a custom header
-func isStandardHeader(headerKey string) bool {
-	key := strings.ToLower(headerKey)
-	for _, header := range standardHeaders {
-		if strings.ToLower(header) == key {
-			return true
-		}
-	}
-	return false
+func isStandardHeader(matchHeaderKey string) bool {
+	return equals(matchHeaderKey, standardHeaders...)
 }
 
 // returns whether object version is a deletemarker and if object qualifies for replication
@@ -175,6 +180,14 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 	if err != nil || rcfg == nil {
 		return false, false, sync
 	}
+	opts := replication.ObjectOpts{
+		Name:         dobj.ObjectName,
+		SSEC:         crypto.SSEC.IsEncrypted(oi.UserDefined),
+		UserTags:     oi.UserTags,
+		DeleteMarker: oi.DeleteMarker,
+		VersionID:    dobj.VersionID,
+	}
+	replicate = rcfg.Replicate(opts)
 	// when incoming delete is removal of a delete marker( a.k.a versioned delete),
 	// GetObjectInfo returns extra information even though it returns errFileNotFound
 	if gerr != nil {
@@ -183,23 +196,20 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 		case replication.Pending, replication.Completed, replication.Failed:
 			validReplStatus = true
 		}
-		if oi.DeleteMarker && validReplStatus {
+		if oi.DeleteMarker && (validReplStatus || replicate) {
 			return oi.DeleteMarker, true, sync
 		}
-		return oi.DeleteMarker, false, sync
+		// can be the case that other cluster is down and duplicate `mc rm --vid`
+		// is issued - this still needs to be replicated back to the other target
+		return oi.DeleteMarker, oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed, sync
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
-	if tgt == nil || tgt.isOffline() {
+	// the target online status should not be used here while deciding
+	// whether to replicate deletes as the target could be temporarily down
+	if tgt == nil {
 		return oi.DeleteMarker, false, false
 	}
-	opts := replication.ObjectOpts{
-		Name:         dobj.ObjectName,
-		SSEC:         crypto.SSEC.IsEncrypted(oi.UserDefined),
-		UserTags:     oi.UserTags,
-		DeleteMarker: oi.DeleteMarker,
-		VersionID:    dobj.VersionID,
-	}
-	return oi.DeleteMarker, rcfg.Replicate(opts), tgt.replicateSync
+	return oi.DeleteMarker, replicate, tgt.replicateSync
 }
 
 // replicate deletes to the designated replication target if replication configuration
@@ -214,24 +224,52 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 // on target.
 func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectAPI ObjectLayer) {
 	bucket := dobj.Bucket
-	rcfg, err := getReplicationConfig(ctx, bucket)
-	if err != nil || rcfg == nil {
-		return
-	}
-	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
-	if tgt == nil {
-		return
-	}
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
 		versionID = dobj.VersionID
 	}
+
+	rcfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil || rcfg == nil {
+		logger.LogIf(ctx, err)
+		sendEvent(eventArgs{
+			BucketName: bucket,
+			Object: ObjectInfo{
+				Bucket:       bucket,
+				Name:         dobj.ObjectName,
+				VersionID:    versionID,
+				DeleteMarker: dobj.DeleteMarker,
+			},
+			Host:      "Internal: [Replication]",
+			EventName: event.ObjectReplicationNotTracked,
+		})
+		return
+	}
+
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
+	if tgt == nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, rcfg.RoleArn))
+		sendEvent(eventArgs{
+			BucketName: bucket,
+			Object: ObjectInfo{
+				Bucket:       bucket,
+				Name:         dobj.ObjectName,
+				VersionID:    versionID,
+				DeleteMarker: dobj.DeleteMarker,
+			},
+			Host:      "Internal: [Replication]",
+			EventName: event.ObjectReplicationNotTracked,
+		})
+		return
+	}
+
 	rmErr := tgt.RemoveObject(ctx, rcfg.GetDestination().Bucket, dobj.ObjectName, miniogo.RemoveObjectOptions{
 		VersionID: versionID,
 		Internal: miniogo.AdvancedRemoveOptions{
 			ReplicationDeleteMarker: dobj.DeleteMarkerVersionID != "",
 			ReplicationMTime:        dobj.DeleteMarkerMTime.Time,
 			ReplicationStatus:       miniogo.ReplicationStatusReplica,
+			ReplicationRequest:      true, // always set this to distinguish between `mc mirror` replication and serverside
 		},
 	})
 
@@ -244,6 +282,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA
 		} else {
 			versionPurgeStatus = Failed
 		}
+		logger.LogIf(ctx, fmt.Errorf("Unable to replicate delete marker to %s/%s(%s): %s", rcfg.GetDestination().Bucket, dobj.ObjectName, versionID, rmErr))
 	} else {
 		if dobj.VersionID == "" {
 			replicationStatus = string(replication.Completed)
@@ -251,79 +290,109 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA
 			versionPurgeStatus = Complete
 		}
 	}
+
 	var eventName = event.ObjectReplicationComplete
 	if replicationStatus == string(replication.Failed) || versionPurgeStatus == Failed {
 		eventName = event.ObjectReplicationFailed
 	}
-	objInfo := ObjectInfo{
-		Name:               dobj.ObjectName,
-		DeleteMarker:       dobj.DeleteMarker,
-		VersionID:          versionID,
-		ReplicationStatus:  replication.StatusType(dobj.DeleteMarkerReplicationStatus),
-		VersionPurgeStatus: versionPurgeStatus,
-	}
-
-	eventArg := &eventArgs{
-		BucketName: bucket,
-		Object:     objInfo,
-		Host:       "Internal: [Replication]",
-		EventName:  eventName,
-	}
-	sendEvent(*eventArg)
 
 	// Update metadata on the delete marker or purge permanent delete if replication success.
-	if _, err = objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{
+	dobjInfo, err := objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{
 		VersionID:                     versionID,
-		DeleteMarker:                  dobj.DeleteMarker,
 		DeleteMarkerReplicationStatus: replicationStatus,
-		Versioned:                     globalBucketVersioningSys.Enabled(bucket),
 		VersionPurgeStatus:            versionPurgeStatus,
+		Versioned:                     globalBucketVersioningSys.Enabled(bucket),
 		VersionSuspended:              globalBucketVersioningSys.Suspended(bucket),
-	}); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s %s: %w", bucket, dobj.ObjectName, dobj.VersionID, err))
+	})
+	if err != nil && !isErrVersionNotFound(err) { // VersionNotFound would be reported by pool that object version is missing on.
+		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %s", bucket, dobj.ObjectName, versionID, err))
+		sendEvent(eventArgs{
+			BucketName: bucket,
+			Object: ObjectInfo{
+				Bucket:       bucket,
+				Name:         dobj.ObjectName,
+				VersionID:    versionID,
+				DeleteMarker: dobj.DeleteMarker,
+			},
+			Host:      "Internal: [Replication]",
+			EventName: eventName,
+		})
+	} else {
+		sendEvent(eventArgs{
+			BucketName: bucket,
+			Object:     dobjInfo,
+			Host:       "Internal: [Replication]",
+			EventName:  eventName,
+		})
 	}
 }
 
 func getCopyObjMetadata(oi ObjectInfo, dest replication.Destination) map[string]string {
 	meta := make(map[string]string, len(oi.UserDefined))
 	for k, v := range oi.UserDefined {
-		if k == xhttp.AmzBucketReplicationStatus {
-			continue
-		}
 		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			continue
 		}
+
+		if equals(k, xhttp.AmzBucketReplicationStatus) {
+			continue
+		}
+
+		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+			continue
+		}
+
 		meta[k] = v
 	}
+
 	if oi.ContentEncoding != "" {
 		meta[xhttp.ContentEncoding] = oi.ContentEncoding
 	}
+
 	if oi.ContentType != "" {
 		meta[xhttp.ContentType] = oi.ContentType
 	}
-	tag, err := tags.ParseObjectTags(oi.UserTags)
-	if err != nil {
-		return nil
-	}
-	if tag != nil {
-		meta[xhttp.AmzObjectTagging] = tag.String()
+
+	if oi.UserTags != "" {
+		meta[xhttp.AmzObjectTagging] = oi.UserTags
 		meta[xhttp.AmzTagDirective] = "REPLACE"
 	}
+
 	sc := dest.StorageClass
 	if sc == "" {
 		sc = oi.StorageClass
 	}
-	meta[xhttp.AmzStorageClass] = sc
-	if oi.UserTags != "" {
-		meta[xhttp.AmzObjectTagging] = oi.UserTags
+	if sc != "" {
+		meta[xhttp.AmzStorageClass] = sc
 	}
-	meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339Nano)
 	meta[xhttp.MinIOSourceETag] = oi.ETag
+	meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339Nano)
 	meta[xhttp.AmzBucketReplicationStatus] = replication.Replica.String()
 	return meta
 }
 
-func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
+type caseInsensitiveMap map[string]string
+
+// Lookup map entry case insensitively.
+func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
+	if len(m) == 0 {
+		return "", false
+	}
+	for _, k := range []string{
+		key,
+		strings.ToLower(key),
+		http.CanonicalHeaderKey(key),
+	} {
+		v, ok := m[k]
+		if ok {
+			return v, ok
+		}
+	}
+	return "", false
+}
+
+func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
 		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
@@ -334,45 +403,58 @@ func putReplicationOpts(ctx context.Context, dest replication.Destination, objIn
 		}
 		meta[k] = v
 	}
-	tag, err := tags.ParseObjectTags(objInfo.UserTags)
-	if err != nil {
-		return
-	}
+
 	sc := dest.StorageClass
 	if sc == "" {
 		sc = objInfo.StorageClass
 	}
 	putOpts = miniogo.PutObjectOptions{
 		UserMetadata:    meta,
-		UserTags:        tag.ToMap(),
 		ContentType:     objInfo.ContentType,
 		ContentEncoding: objInfo.ContentEncoding,
 		StorageClass:    sc,
 		Internal: miniogo.AdvancedPutOptions{
-			SourceVersionID:   objInfo.VersionID,
-			ReplicationStatus: miniogo.ReplicationStatusReplica,
-			SourceMTime:       objInfo.ModTime,
-			SourceETag:        objInfo.ETag,
+			SourceVersionID:    objInfo.VersionID,
+			ReplicationStatus:  miniogo.ReplicationStatusReplica,
+			SourceMTime:        objInfo.ModTime,
+			SourceETag:         objInfo.ETag,
+			ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
 		},
 	}
-	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
+	if objInfo.UserTags != "" {
+		tag, _ := tags.ParseObjectTags(objInfo.UserTags)
+		if tag != nil {
+			putOpts.UserTags = tag.ToMap()
+		}
+	}
+
+	lkMap := caseInsensitiveMap(objInfo.UserDefined)
+	if lang, ok := lkMap.Lookup(xhttp.ContentLanguage); ok {
+		putOpts.ContentLanguage = lang
+	}
+	if disp, ok := lkMap.Lookup(xhttp.ContentDisposition); ok {
+		putOpts.ContentDisposition = disp
+	}
+	if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok {
+		putOpts.CacheControl = cc
+	}
+	if mode, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); ok {
 		rmode := miniogo.RetentionMode(mode)
 		putOpts.Mode = rmode
 	}
-	if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
-		rdate, err := time.Parse(time.RFC3339Nano, retainDateStr)
+	if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
+		rdate, err := time.Parse(time.RFC3339, retainDateStr)
 		if err != nil {
-			return
+			return putOpts, err
 		}
 		putOpts.RetainUntilDate = rdate
 	}
-	if lhold, ok := objInfo.UserDefined[xhttp.AmzObjectLockLegalHold]; ok {
+	if lhold, ok := lkMap.Lookup(xhttp.AmzObjectLockLegalHold); ok {
 		putOpts.LegalHold = miniogo.LegalHoldStatus(lhold)
 	}
 	if crypto.S3.IsEncrypted(objInfo.UserDefined) {
 		putOpts.ServerSideEncryption = encrypt.NewSSE()
 	}
-
 	return
 }
 
@@ -384,113 +466,230 @@ const (
 	replicateAll      replicationAction = "all"
 )
 
+// matches k1 with all keys, returns 'true' if one of them matches
+func equals(k1 string, keys ...string) bool {
+	for _, k2 := range keys {
+		if strings.ToLower(k1) == strings.ToLower(k2) {
+			return true
+		}
+	}
+	return false
+}
+
 // returns replicationAction by comparing metadata between source and target
 func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo) replicationAction {
 	// needs full replication
 	if oi1.ETag != oi2.ETag ||
 		oi1.VersionID != oi2.VersionID ||
 		oi1.Size != oi2.Size ||
-		oi1.DeleteMarker != oi2.IsDeleteMarker {
+		oi1.DeleteMarker != oi2.IsDeleteMarker ||
+		oi1.ModTime.Unix() != oi2.LastModified.Unix() {
 		return replicateAll
 	}
 
-	if !oi1.ModTime.Equal(oi2.LastModified) ||
-		oi1.ContentType != oi2.ContentType ||
-		oi1.StorageClass != oi2.StorageClass {
+	if oi1.ContentType != oi2.ContentType {
 		return replicateMetadata
 	}
+
 	if oi1.ContentEncoding != "" {
-		enc, ok := oi2.UserMetadata[xhttp.ContentEncoding]
-		if !ok || enc != oi1.ContentEncoding {
+		enc, ok := oi2.Metadata[xhttp.ContentEncoding]
+		if !ok {
+			enc, ok = oi2.Metadata[strings.ToLower(xhttp.ContentEncoding)]
+			if !ok {
+				return replicateMetadata
+			}
+		}
+		if strings.Join(enc, ",") != oi1.ContentEncoding {
 			return replicateMetadata
 		}
 	}
-	for k, v := range oi2.UserMetadata {
-		oi2.Metadata[k] = []string{v}
-	}
-	if len(oi2.Metadata) != len(oi1.UserDefined) {
+
+	t, _ := tags.ParseObjectTags(oi1.UserTags)
+	if !reflect.DeepEqual(oi2.UserTags, t.ToMap()) {
 		return replicateMetadata
 	}
-	for k1, v1 := range oi1.UserDefined {
-		if v2, ok := oi2.Metadata[k1]; !ok || v1 != strings.Join(v2, "") {
-			return replicateMetadata
+
+	// Compare only necessary headers
+	compareKeys := []string{
+		"Expires",
+		"Cache-Control",
+		"Content-Language",
+		"Content-Disposition",
+		"X-Amz-Object-Lock-Mode",
+		"X-Amz-Object-Lock-Retain-Until-Date",
+		"X-Amz-Object-Lock-Legal-Hold",
+		"X-Amz-Website-Redirect-Location",
+		"X-Amz-Meta-",
+	}
+
+	// compare metadata on both maps to see if meta is identical
+	compareMeta1 := make(map[string]string)
+	for k, v := range oi1.UserDefined {
+		var found bool
+		for _, prefix := range compareKeys {
+			if !strings.HasPrefix(strings.ToLower(k), strings.ToLower(prefix)) {
+				continue
+			}
+			found = true
+			break
+		}
+		if found {
+			compareMeta1[strings.ToLower(k)] = v
 		}
 	}
-	t, _ := tags.MapToObjectTags(oi2.UserTags)
-	if t.String() != oi1.UserTags {
+
+	compareMeta2 := make(map[string]string)
+	for k, v := range oi2.Metadata {
+		var found bool
+		for _, prefix := range compareKeys {
+			if !strings.HasPrefix(strings.ToLower(k), strings.ToLower(prefix)) {
+				continue
+			}
+			found = true
+			break
+		}
+		if found {
+			compareMeta2[strings.ToLower(k)] = strings.Join(v, ",")
+		}
+	}
+
+	if !reflect.DeepEqual(compareMeta1, compareMeta2) {
 		return replicateMetadata
 	}
+
 	return replicateNone
 }
 
 // replicateObject replicates the specified version of the object to destination bucket
 // The source object is then updated to reflect the replication status.
 func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLayer) {
+	z, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		return
+	}
+
 	bucket := objInfo.Bucket
 	object := objInfo.Name
 
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
 		return
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil {
 		logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, cfg.RoleArn))
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
 		return
 	}
 	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
 		VersionID: objInfo.VersionID,
 	})
 	if err != nil {
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
 		logger.LogIf(ctx, err)
 		return
 	}
+	defer gr.Close() // hold read lock for entire transaction
+
 	objInfo = gr.ObjInfo
 	size, err := objInfo.GetActualSize()
 	if err != nil {
 		logger.LogIf(ctx, err)
-		gr.Close()
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
 		return
 	}
 
 	dest := cfg.GetDestination()
 	if dest.Bucket == "" {
-		gr.Close()
+		logger.LogIf(ctx, fmt.Errorf("Unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
 		return
 	}
 
 	rtype := replicateAll
-	oi, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID})
+	oi, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{
+		VersionID: objInfo.VersionID,
+		Internal: miniogo.AdvancedGetOptions{
+			ReplicationProxyRequest: "false",
+		}})
 	if err == nil {
 		rtype = getReplicationAction(objInfo, oi)
 		if rtype == replicateNone {
-			gr.Close()
 			// object with same VersionID already exists, replication kicked off by
 			// PutObject might have completed.
 			return
 		}
 	}
 	replicationStatus := replication.Completed
+	// use core client to avoid doing multipart on PUT
+	c := &miniogo.Core{Client: tgt.Client}
 	if rtype != replicateAll {
-		gr.Close()
-
 		// replicate metadata for object tagging/copy with metadata replacement
-		dstOpts := miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{SourceVersionID: objInfo.VersionID}}
-		c := &miniogo.Core{Client: tgt.Client}
-		_, err = c.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), dstOpts)
-		if err != nil {
+		srcOpts := miniogo.CopySrcOptions{
+			Bucket:    dest.Bucket,
+			Object:    object,
+			VersionID: objInfo.VersionID}
+		dstOpts := miniogo.PutObjectOptions{
+			Internal: miniogo.AdvancedPutOptions{
+				SourceVersionID:    objInfo.VersionID,
+				ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
+			}}
+		if _, err = c.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), srcOpts, dstOpts); err != nil {
 			replicationStatus = replication.Failed
+			logger.LogIf(ctx, fmt.Errorf("Unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 		}
 	} else {
 		target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
-			gr.Close()
+			sendEvent(eventArgs{
+				EventName:  event.ObjectReplicationNotTracked,
+				BucketName: bucket,
+				Object:     objInfo,
+				Host:       "Internal: [Replication]",
+			})
+			return
+		}
+
+		putOpts, err := putReplicationOpts(ctx, dest, objInfo)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%w", bucket, cfg.RoleArn, err))
+			sendEvent(eventArgs{
+				EventName:  event.ObjectReplicationNotTracked,
+				BucketName: bucket,
+				Object:     objInfo,
+				Host:       "Internal: [Replication]",
+			})
 			return
 		}
 
-		putOpts := putReplicationOpts(ctx, dest, objInfo)
 		// Setup bandwidth throttling
 		peers, _ := globalEndpoints.peers()
 		totalNodesCount := len(peers)
@@ -505,39 +704,43 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 
 		// r takes over closing gr.
 		r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
-		_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, putOpts)
-		if err != nil {
+		if _, err = c.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts); err != nil {
 			replicationStatus = replication.Failed
+			logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %w", bucket, objInfo.Name, objInfo.VersionID, err))
 		}
-		r.Close()
+		defer r.Close()
 	}
+
 	objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
 	if objInfo.UserTags != "" {
 		objInfo.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
 
 	// FIXME: add support for missing replication events
-	// - event.ObjectReplicationNotTracked
 	// - event.ObjectReplicationMissedThreshold
 	// - event.ObjectReplicationReplicatedAfterThreshold
 	var eventName = event.ObjectReplicationComplete
 	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
 	}
+
+	// This lower level implementation is necessary to avoid write locks from CopyObject.
+	poolIdx, err := z.getPoolIdx(ctx, bucket, object, objInfo.Size)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w", bucket, objInfo.Name, objInfo.VersionID, err))
+	} else {
+		if err = z.serverPools[poolIdx].getHashedSet(object).updateObjectMeta(ctx, bucket, object, objInfo.UserDefined, ObjectOptions{
+			VersionID: objInfo.VersionID,
+		}); err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w", bucket, objInfo.Name, objInfo.VersionID, err))
+		}
+	}
 	sendEvent(eventArgs{
 		EventName:  eventName,
 		BucketName: bucket,
 		Object:     objInfo,
 		Host:       "Internal: [Replication]",
 	})
-	objInfo.metadataOnly = true // Perform only metadata updates.
-	if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{
-		VersionID: objInfo.VersionID,
-	}, ObjectOptions{
-		VersionID: objInfo.VersionID,
-	}); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s: %s", objInfo.VersionID, err))
-	}
 }
 
 // filterReplicationStatusMetadata filters replication status metadata for COPY
@@ -568,91 +771,102 @@ type DeletedObjectVersionInfo struct {
 	DeletedObject
 	Bucket string
 }
-type replicationState struct {
-	// add future metrics here
-	replicaCh       chan ObjectInfo
-	replicaDeleteCh chan DeletedObjectVersionInfo
-}
-
-func (r *replicationState) queueReplicaTask(oi ObjectInfo) {
-	if r == nil {
-		return
-	}
-	select {
-	case r.replicaCh <- oi:
-	default:
-	}
-}
-
-func (r *replicationState) queueReplicaDeleteTask(doi DeletedObjectVersionInfo) {
-	if r == nil {
-		return
-	}
-	select {
-	case r.replicaDeleteCh <- doi:
-	default:
-	}
-}
 
 var (
-	globalReplicationState *replicationState
-	// TODO: currently keeping it conservative
-	// but eventually can be tuned in future,
-	// take only half the CPUs for replication
-	// conservatively.
-	globalReplicationConcurrent = runtime.GOMAXPROCS(0) / 2
+	globalReplicationPool *ReplicationPool
 )
 
-func newReplicationState() *replicationState {
-
-	// fix minimum concurrent replication to 1 for single CPU setup
-	if globalReplicationConcurrent == 0 {
-		globalReplicationConcurrent = 1
-	}
-	rs := &replicationState{
-		replicaCh:       make(chan ObjectInfo, 10000),
-		replicaDeleteCh: make(chan DeletedObjectVersionInfo, 10000),
-	}
-	go func() {
-		<-GlobalContext.Done()
-		close(rs.replicaCh)
-		close(rs.replicaDeleteCh)
-	}()
-	return rs
+// ReplicationPool describes replication pool
+type ReplicationPool struct {
+	mu              sync.Mutex
+	size            int
+	replicaCh       chan ObjectInfo
+	replicaDeleteCh chan DeletedObjectVersionInfo
+	killCh          chan struct{}
+	wg              sync.WaitGroup
+	ctx             context.Context
+	objLayer        ObjectLayer
 }
 
-// addWorker creates a new worker to process tasks
-func (r *replicationState) addWorker(ctx context.Context, objectAPI ObjectLayer) {
-	// Add a new worker.
+// NewReplicationPool creates a pool of replication workers of specified size
+func NewReplicationPool(ctx context.Context, o ObjectLayer, sz int) *ReplicationPool {
+	pool := &ReplicationPool{
+		replicaCh:       make(chan ObjectInfo, 10000),
+		replicaDeleteCh: make(chan DeletedObjectVersionInfo, 10000),
+		ctx:             ctx,
+		objLayer:        o,
+	}
 	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case oi, ok := <-r.replicaCh:
-				if !ok {
-					return
-				}
-				replicateObject(ctx, oi, objectAPI)
-			case doi, ok := <-r.replicaDeleteCh:
-				if !ok {
-					return
-				}
-				replicateDelete(ctx, doi, objectAPI)
-			}
-		}
+		<-ctx.Done()
+		close(pool.replicaCh)
+		close(pool.replicaDeleteCh)
 	}()
+	pool.Resize(sz)
+	return pool
+}
+
+// AddWorker adds a replication worker to the pool
+func (p *ReplicationPool) AddWorker() {
+	defer p.wg.Done()
+	for {
+		select {
+		case <-p.ctx.Done():
+			return
+		case oi, ok := <-p.replicaCh:
+			if !ok {
+				return
+			}
+			replicateObject(p.ctx, oi, p.objLayer)
+		case doi, ok := <-p.replicaDeleteCh:
+			if !ok {
+				return
+			}
+			replicateDelete(p.ctx, doi, p.objLayer)
+		case <-p.killCh:
+			return
+		}
+	}
+
+}
+
+//Resize replication pool to new size
+func (p *ReplicationPool) Resize(n int) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	for p.size < n {
+		p.size++
+		p.wg.Add(1)
+		go p.AddWorker()
+	}
+	for p.size > n {
+		p.size--
+		go func() { p.killCh <- struct{}{} }()
+	}
+}
+
+func (p *ReplicationPool) queueReplicaTask(oi ObjectInfo) {
+	if p == nil {
+		return
+	}
+	select {
+	case p.replicaCh <- oi:
+	default:
+	}
+}
+
+func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectVersionInfo) {
+	if p == nil {
+		return
+	}
+	select {
+	case p.replicaDeleteCh <- doi:
+	default:
+	}
 }
 
 func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
-	if globalReplicationState == nil {
-		return
-	}
-
-	// Start with globalReplicationConcurrent.
-	for i := 0; i < globalReplicationConcurrent; i++ {
-		globalReplicationState.addWorker(ctx, objectAPI)
-	}
+	globalReplicationPool = NewReplicationPool(ctx, objectAPI, globalAPIConfig.getReplicationWorkers())
 }
 
 // get Reader from replication target if active-active replication is in place and
@@ -670,7 +884,7 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 		VersionID:            opts.VersionID,
 		ServerSideEncryption: opts.ServerSideEncryption,
 		Internal: miniogo.AdvancedGetOptions{
-			ReplicationProxyRequest: true,
+			ReplicationProxyRequest: "true",
 		},
 	}
 	// get correct offsets for encrypted object
@@ -679,8 +893,11 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 			return nil, false
 		}
 	}
+	// Make sure to match ETag when proxying.
+	if err = gopts.SetMatchETag(oi.ETag); err != nil {
+		return nil, false
+	}
 	c := miniogo.Core{Client: tgt.Client}
-
 	obj, _, _, err := c.GetObject(ctx, bucket, object, gopts)
 	if err != nil {
 		return nil, false
@@ -691,6 +908,7 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 	if err != nil {
 		return nil, false
 	}
+	reader.ObjInfo = oi.Clone()
 	return reader, true
 }
 
@@ -703,10 +921,11 @@ func isProxyable(ctx context.Context, bucket string) bool {
 	dest := cfg.GetDestination()
 	return dest.Bucket == bucket
 }
+
 func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions) (tgt *TargetClient, oi ObjectInfo, proxy bool, err error) {
 	// this option is set when active-active replication is in place between site A -> B,
 	// and site B does not have the object yet.
-	if opts.ProxyRequest { // true only when site B sets MinIOSourceProxyRequest header
+	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
 		return nil, oi, false, nil
 	}
 	cfg, err := getReplicationConfig(ctx, bucket)
@@ -730,14 +949,14 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 	}
 	tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil || tgt.isOffline() {
-		return nil, oi, false, fmt.Errorf("missing target")
+		return nil, oi, false, fmt.Errorf("target is offline or not configured")
 	}
 
 	gopts := miniogo.GetObjectOptions{
 		VersionID:            opts.VersionID,
 		ServerSideEncryption: opts.ServerSideEncryption,
 		Internal: miniogo.AdvancedGetOptions{
-			ReplicationProxyRequest: true,
+			ReplicationProxyRequest: "true",
 		},
 	}
 
@@ -745,6 +964,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 	if err != nil {
 		return nil, oi, false, err
 	}
+
 	tags, _ := tags.MapToObjectTags(objInfo.UserTags)
 	oi = ObjectInfo{
 		Bucket:            bucket,
@@ -759,12 +979,18 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 		Expires:           objInfo.Expires,
 		StorageClass:      objInfo.StorageClass,
 		ReplicationStatus: replication.StatusType(objInfo.ReplicationStatus),
-		UserDefined:       cloneMSS(objInfo.UserMetadata),
 		UserTags:          tags.String(),
 	}
-	if ce, ok := oi.UserDefined[xhttp.ContentEncoding]; ok {
+	oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
+	for k, v := range objInfo.Metadata {
+		oi.UserDefined[k] = v[0]
+	}
+	ce, ok := oi.UserDefined[xhttp.ContentEncoding]
+	if !ok {
+		ce, ok = oi.UserDefined[strings.ToLower(xhttp.ContentEncoding)]
+	}
+	if ok {
 		oi.ContentEncoding = ce
-		delete(oi.UserDefined, xhttp.ContentEncoding)
 	}
 	return tgt, oi, true, nil
 }
@@ -780,7 +1006,7 @@ func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer,
 	if sync {
 		replicateObject(ctx, objInfo, o)
 	} else {
-		globalReplicationState.queueReplicaTask(objInfo)
+		globalReplicationPool.queueReplicaTask(objInfo)
 	}
 }
 
@@ -788,6 +1014,6 @@ func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectVersionInfo,
 	if sync {
 		replicateDelete(ctx, dv, o)
 	} else {
-		globalReplicationState.queueReplicaDeleteTask(dv)
+		globalReplicationPool.queueReplicaDeleteTask(dv)
 	}
 }

cmd/bucket-targets.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"net/http"
@@ -32,11 +33,10 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/madmin"
-	sha256 "github.com/minio/sha256-simd"
 )
 
 const (
-	defaultHealthCheckDuration = 60 * time.Second
+	defaultHealthCheckDuration = 100 * time.Second
 )
 
 // BucketTargetSys represents bucket targets subsystem
@@ -362,6 +362,7 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	api, err := minio.New(tcfg.Endpoint, &miniogo.Options{
 		Creds:     creds,
 		Secure:    tcfg.Secure,
+		Region:    tcfg.Region,
 		Transport: getRemoteTargetInstanceTransport,
 	})
 	if err != nil {
@@ -461,10 +462,10 @@ func (tc *TargetClient) healthCheck() {
 		_, err := tc.BucketExists(GlobalContext, tc.bucket)
 		if err != nil {
 			atomic.StoreInt32(&tc.up, 0)
-			time.Sleep(tc.healthCheckDuration * time.Second)
+			time.Sleep(tc.healthCheckDuration)
 			continue
 		}
 		atomic.StoreInt32(&tc.up, 1)
-		time.Sleep(tc.healthCheckDuration * time.Second)
+		time.Sleep(tc.healthCheckDuration)
 	}
 }

cmd/bucket-versioning-handler.go

@@ -40,7 +40,7 @@ const (
 func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketVersioning")
 
-	defer logger.AuditLog(w, r, "PutBucketVersioning", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -98,7 +98,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketVersioning")
 
-	defer logger.AuditLog(w, r, "GetBucketVersioning", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/common-main.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"crypto/x509"
 	"encoding/gob"
 	"errors"
@@ -27,6 +28,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"time"
 
@@ -41,21 +43,36 @@ import (
 	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/handlers"
 )
 
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
 
 func init() {
+	rand.Seed(time.Now().UTC().UnixNano())
+
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	rand.Seed(time.Now().UTC().UnixNano())
+	// Inject into config package.
+	config.Logger.Info = logger.Info
+	config.Logger.LogIf = logger.LogIf
+
 	globalDNSCache = xhttp.NewDNSCache(10*time.Second, 10*time.Second, logger.LogOnceIf)
 
 	initGlobalContext()
 
-	globalReplicationState = newReplicationState()
+	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
+		PassHost:     true,
+		RoundTripper: newGatewayHTTPTransport(1 * time.Hour),
+		Logger: func(err error) {
+			if err != nil && !errors.Is(err, context.Canceled) {
+				logger.LogIf(GlobalContext, err)
+			}
+		},
+	})
+
 	globalTransitionState = newTransitionState()
 
 	console.SetColor("Debug", color.New())
@@ -241,6 +258,14 @@ func handleCommonEnvVars() {
 			}
 			globalDomainNames = append(globalDomainNames, domainName)
 		}
+		sort.Strings(globalDomainNames)
+		lcpSuf := lcpSuffix(globalDomainNames)
+		for _, domainName := range globalDomainNames {
+			if domainName == lcpSuf && len(globalDomainNames) > 1 {
+				logger.Fatal(config.ErrOverlappingDomainValue(nil).Msg("Overlapping domains `%s` not allowed", globalDomainNames),
+					"Invalid MINIO_DOMAIN value in environment variable")
+			}
+		}
 	}
 
 	publicIPs := env.Get(config.EnvPublicIPs, "")

cmd/config-common.go

@@ -20,6 +20,8 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"io/ioutil"
+	"net/http"
 
 	"github.com/minio/minio/pkg/hash"
 )
@@ -27,9 +29,9 @@ import (
 var errConfigNotFound = errors.New("config file not found")
 
 func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
-	var buffer bytes.Buffer
 	// Read entire content by setting size to -1
-	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
+	r, err := objAPI.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
+	if err != nil {
 		// Treat object not found as config not found.
 		if isErrObjectNotFound(err) {
 			return nil, errConfigNotFound
@@ -37,13 +39,16 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 
 		return nil, err
 	}
+	defer r.Close()
 
-	// Return config not found on empty content.
-	if buffer.Len() == 0 {
+	buf, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	if len(buf) == 0 {
 		return nil, errConfigNotFound
 	}
-
-	return buffer.Bytes(), nil
+	return buf, nil
 }
 
 type objectDeleter interface {
@@ -59,12 +64,12 @@ func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string)
 }
 
 func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
-	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)), globalCLIContext.StrictS3Compat)
+	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
 	if err != nil {
 		return err
 	}
 
-	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), ObjectOptions{})
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), ObjectOptions{MaxParity: true})
 	return err
 }
 

cmd/config-current.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"strings"
 	"sync"
@@ -26,7 +27,6 @@ import (
 	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
-	"github.com/minio/minio/cmd/config/crawler"
 	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/config/heal"
@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/notify"
 	"github.com/minio/minio/cmd/config/policy/opa"
+	"github.com/minio/minio/cmd/config/scanner"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -59,7 +60,7 @@ func initHelp() {
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
 		config.HealSubSys:           heal.DefaultKVS,
-		config.CrawlerSubSys:        crawler.DefaultKVS,
+		config.ScannerSubSys:        scanner.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -116,8 +117,8 @@ func initHelp() {
 			Description: "manage object healing frequency and bitrot verification checks",
 		},
 		config.HelpKV{
-			Key:         config.CrawlerSubSys,
-			Description: "manage crawling for usage calculation, lifecycle, healing and more",
+			Key:         config.ScannerSubSys,
+			Description: "manage namespace scanning for usage calculation, lifecycle, healing and more",
 		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
@@ -199,7 +200,7 @@ func initHelp() {
 		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
 		config.HealSubSys:           heal.Help,
-		config.CrawlerSubSys:        crawler.Help,
+		config.ScannerSubSys:        scanner.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.PolicyOPASubSys:      opa.Help,
@@ -228,7 +229,7 @@ var (
 	globalServerConfigMu sync.RWMutex
 )
 
-func validateConfig(s config.Config, setDriveCount int) error {
+func validateConfig(s config.Config, setDriveCounts []int) error {
 	// We must have a global lock for this so nobody else modifies env while we do.
 	defer env.LockSetEnv()()
 
@@ -251,8 +252,10 @@ func validateConfig(s config.Config, setDriveCount int) error {
 	}
 
 	if globalIsErasure {
-		if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
-			return err
+		for _, setDriveCount := range setDriveCounts {
+			if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -271,11 +274,11 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		}
 	}
 
-	if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
+	if _, err = heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
 		return err
 	}
 
-	if _, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default]); err != nil {
+	if _, err = scanner.LookupConfig(s[config.ScannerSubSys][config.Default]); err != nil {
 		return err
 	}
 
@@ -293,7 +296,10 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		}
 	}
 	{
-		kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
+		kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), newCustomHTTPTransportWithHTTP2(
+			&tls.Config{
+				RootCAs: globalRootCAs,
+			}, defaultDialTimeout)())
 		if err != nil {
 			return err
 		}
@@ -342,7 +348,7 @@ func validateConfig(s config.Config, setDriveCount int) error {
 	return notify.TestNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs())
 }
 
-func lookupConfigs(s config.Config, setDriveCount int) {
+func lookupConfigs(s config.Config, setDriveCounts []int) {
 	ctx := GlobalContext
 
 	var err error
@@ -429,7 +435,7 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 	}
 
-	globalAPIConfig.init(apiConfig, setDriveCount)
+	globalAPIConfig.init(apiConfig, setDriveCounts)
 
 	// Initialize remote instance transport once.
 	getRemoteInstanceTransportOnce.Do(func() {
@@ -437,9 +443,17 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 	})
 
 	if globalIsErasure {
-		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
+		for i, setDriveCount := range setDriveCounts {
+			sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
+			if err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
+				break
+			}
+			// if we validated all setDriveCounts and it was successful
+			// proceed to store the correct storage class globally.
+			if i == len(setDriveCounts)-1 {
+				globalStorageClass = sc
+			}
 		}
 	}
 
@@ -461,7 +475,10 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		}
 	}
 
-	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
+	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), newCustomHTTPTransportWithHTTP2(
+		&tls.Config{
+			RootCAs: globalRootCAs,
+		}, defaultDialTimeout)())
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to setup KMS config: %w", err))
 	}
@@ -470,11 +487,13 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to setup KMS with current KMS config: %w", err))
 	}
+	globalAutoEncryption = kmsCfg.AutoEncryption // Enable auto-encryption if enabled
 
-	// Enable auto-encryption if enabled
-	globalAutoEncryption = kmsCfg.AutoEncryption
-	if globalAutoEncryption && !globalIsGateway {
-		logger.LogIf(ctx, fmt.Errorf("%s env is deprecated please migrate to using `mc encrypt` at bucket level", crypto.EnvKMSAutoEncryption))
+	if kmsCfg.Vault.Enabled {
+		const deprecationWarning = `Native Hashicorp Vault support is deprecated and will be removed on 2021-10-01. Please migrate to KES + Hashicorp Vault: https://github.com/minio/kes/wiki/Hashicorp-Vault-Keystore
+Note that native Hashicorp Vault and KES + Hashicorp Vault are not compatible.
+If you need help to migrate smoothly visit: https://min.io/pricing`
+		logger.LogIf(ctx, fmt.Errorf(deprecationWarning))
 	}
 
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
@@ -534,7 +553,7 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 					http.WithAuthToken(l.AuthToken),
 					http.WithUserAgent(loggerUserAgent),
 					http.WithLogKind(string(logger.All)),
-					http.WithTransport(NewGatewayHTTPTransport()),
+					http.WithTransport(NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)),
 				),
 			); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit HTTP target: %w", err))
@@ -553,12 +572,16 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 	}
 
 	// Apply dynamic config values
-	logger.LogIf(ctx, applyDynamicConfig(ctx, s))
+	logger.LogIf(ctx, applyDynamicConfig(ctx, newObjectLayerFn(), s))
 }
 
 // applyDynamicConfig will apply dynamic config values.
 // Dynamic systems should be in config.SubSystemsDynamic as well.
-func applyDynamicConfig(ctx context.Context, s config.Config) error {
+func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config) error {
+	if objAPI == nil {
+		return nil
+	}
+
 	// Read all dynamic configs.
 	// API
 	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
@@ -571,28 +594,27 @@ func applyDynamicConfig(ctx context.Context, s config.Config) error {
 	if err != nil {
 		return fmt.Errorf("Unable to setup Compression: %w", err)
 	}
-	objAPI := newObjectLayerFn()
-	if objAPI != nil {
-		if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
-			return fmt.Errorf("Backend does not support compression")
-		}
+
+	// Validate if the object layer supports compression.
+	if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
+		return fmt.Errorf("Backend does not support compression")
 	}
 
 	// Heal
 	healCfg, err := heal.LookupConfig(s[config.HealSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to apply heal config: %w", err))
+		return fmt.Errorf("Unable to apply heal config: %w", err)
 	}
 
-	// Crawler
-	crawlerCfg, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default])
+	// Scanner
+	scannerCfg, err := scanner.LookupConfig(s[config.ScannerSubSys][config.Default])
 	if err != nil {
-		return fmt.Errorf("Unable to apply crawler config: %w", err)
+		return fmt.Errorf("Unable to apply scanner config: %w", err)
 	}
 
 	// Apply configurations.
 	// We should not fail after this.
-	globalAPIConfig.init(apiConfig, globalAPIConfig.setDriveCount)
+	globalAPIConfig.init(apiConfig, objAPI.SetDriveCounts())
 
 	globalCompressConfigMu.Lock()
 	globalCompressConfig = cmpCfg
@@ -602,7 +624,7 @@ func applyDynamicConfig(ctx context.Context, s config.Config) error {
 	globalHealConfig = healCfg
 	globalHealConfigMu.Unlock()
 
-	logger.LogIf(ctx, crawlerSleeper.Update(crawlerCfg.Delay, crawlerCfg.MaxWait))
+	logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 
 	// Update all dynamic config values in memory.
 	globalServerConfigMu.Lock()
@@ -723,7 +745,7 @@ func loadConfig(objAPI ObjectLayer) error {
 	}
 
 	// Override any values from ENVs.
-	lookupConfigs(srvCfg, objAPI.SetDriveCount())
+	lookupConfigs(srvCfg, objAPI.SetDriveCounts())
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()

cmd/config/api/api.go

@@ -29,14 +29,14 @@ import (
 
 // API sub-system constants
 const (
-	apiRequestsMax             = "requests_max"
-	apiRequestsDeadline        = "requests_deadline"
-	apiClusterDeadline         = "cluster_deadline"
-	apiCorsAllowOrigin         = "cors_allow_origin"
-	apiRemoteTransportDeadline = "remote_transport_deadline"
-	apiListQuorum              = "list_quorum"
-	apiExtendListCacheLife     = "extend_list_cache_life"
-
+	apiRequestsMax                = "requests_max"
+	apiRequestsDeadline           = "requests_deadline"
+	apiClusterDeadline            = "cluster_deadline"
+	apiCorsAllowOrigin            = "cors_allow_origin"
+	apiRemoteTransportDeadline    = "remote_transport_deadline"
+	apiListQuorum                 = "list_quorum"
+	apiExtendListCacheLife        = "extend_list_cache_life"
+	apiReplicationWorkers         = "replication_workers"
 	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
 	EnvAPIClusterDeadline         = "MINIO_API_CLUSTER_DEADLINE"
@@ -45,6 +45,7 @@ const (
 	EnvAPIListQuorum              = "MINIO_API_LIST_QUORUM"
 	EnvAPIExtendListCacheLife     = "MINIO_API_EXTEND_LIST_CACHE_LIFE"
 	EnvAPISecureCiphers           = "MINIO_API_SECURE_CIPHERS"
+	EnvAPIReplicationWorkers      = "MINIO_API_REPLICATION_WORKERS"
 )
 
 // Deprecated key and ENVs
@@ -78,12 +79,16 @@ var (
 		},
 		config.KV{
 			Key:   apiListQuorum,
-			Value: "optimal",
+			Value: "strict",
 		},
 		config.KV{
 			Key:   apiExtendListCacheLife,
 			Value: "0s",
 		},
+		config.KV{
+			Key:   apiReplicationWorkers,
+			Value: "100",
+		},
 	}
 )
 
@@ -96,6 +101,7 @@ type Config struct {
 	RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
 	ListQuorum              string        `json:"list_strict_quorum"`
 	ExtendListLife          time.Duration `json:"extend_list_cache_life"`
+	ReplicationWorkers      int           `json:"replication_workers"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -173,6 +179,15 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
+	replicationWorkers, err := strconv.Atoi(env.Get(EnvAPIReplicationWorkers, kvs.Get(apiReplicationWorkers)))
+	if err != nil {
+		return cfg, err
+	}
+
+	if replicationWorkers <= 0 {
+		return cfg, config.ErrInvalidReplicationWorkersValue(nil).Msg("Minimum number of replication workers should be 1")
+	}
+
 	return Config{
 		RequestsMax:             requestsMax,
 		RequestsDeadline:        requestsDeadline,
@@ -181,5 +196,6 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		RemoteTransportDeadline: remoteTransportDeadline,
 		ListQuorum:              listQuorum,
 		ExtendListLife:          listLife,
+		ReplicationWorkers:      replicationWorkers,
 	}, nil
 }

cmd/config/api/help.go

@@ -45,5 +45,11 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
+		config.HelpKV{
+			Key:         apiReplicationWorkers,
+			Description: `set the number of replication workers, defaults to 100`,
+			Optional:    true,
+			Type:        "number",
+		},
 	}
 )

cmd/config/cache/help.go

@@ -40,19 +40,13 @@ var (
 		},
 		config.HelpKV{
 			Key:         Exclude,
-			Description: `comma separated wildcard exclusion patterns e.g. "bucket/*.tmp,*.exe"`,
+			Description: `exclude cache for following patterns e.g. "bucket/*.tmp,*.exe"`,
 			Optional:    true,
 			Type:        "csv",
 		},
-		config.HelpKV{
-			Key:         config.Comment,
-			Description: config.DefaultComment,
-			Optional:    true,
-			Type:        "sentence",
-		},
 		config.HelpKV{
 			Key:         After,
-			Description: `minimum accesses before caching an object`,
+			Description: `minimum number of access before caching an object`,
 			Optional:    true,
 			Type:        "number",
 		},
@@ -80,5 +74,11 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         config.Comment,
+			Description: config.DefaultComment,
+			Optional:    true,
+			Type:        "sentence",
+		},
 	}
 )

cmd/config/certs.go

@@ -23,6 +23,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
+	"errors"
 	"io/ioutil"
 
 	"github.com/minio/minio/pkg/env"
@@ -113,3 +114,12 @@ func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	}
 	return cert, nil
 }
+
+// EnsureCertAndKey checks if both client certificate and key paths are provided
+func EnsureCertAndKey(ClientCert, ClientKey string) error {
+	if (ClientCert != "" && ClientKey == "") ||
+		(ClientCert == "" && ClientKey != "") {
+		return errors.New("cert and key must be specified as a pair")
+	}
+	return nil
+}

cmd/config/compress/compress.go

@@ -45,7 +45,7 @@ const (
 
 	// Include-list for compression.
 	DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
-	DefaultMimeTypes  = "text/*,application/json,application/xml"
+	DefaultMimeTypes  = "text/*,application/json,application/xml,binary/octet-stream"
 )
 
 // DefaultKVS - default KV config for compression settings

cmd/config/config.go

@@ -77,6 +77,7 @@ const (
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
 	HealSubSys           = "heal"
+	ScannerSubSys        = "scanner"
 	CrawlerSubSys        = "crawler"
 
 	// Add new constants here if you add new fields to config.
@@ -114,7 +115,7 @@ var SubSystems = set.CreateStringSet(
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
-	CrawlerSubSys,
+	ScannerSubSys,
 	HealSubSys,
 	NotifyAMQPSubSys,
 	NotifyESSubSys,
@@ -132,7 +133,7 @@ var SubSystems = set.CreateStringSet(
 var SubSystemsDynamic = set.CreateStringSet(
 	APISubSys,
 	CompressionSubSys,
-	CrawlerSubSys,
+	ScannerSubSys,
 	HealSubSys,
 )
 
@@ -151,7 +152,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
 	HealSubSys,
-	CrawlerSubSys,
+	ScannerSubSys,
 }...)
 
 // Constant separators
@@ -462,6 +463,13 @@ func LookupWorm() (bool, error) {
 	return ParseBool(env.Get(EnvWorm, EnableOff))
 }
 
+// Carries all the renamed sub-systems from their
+// previously known names
+var renamedSubsys = map[string]string{
+	CrawlerSubSys: ScannerSubSys,
+	// Add future sub-system renames
+}
+
 // Merge - merges a new config with all the
 // missing values for default configs,
 // returns a config.
@@ -477,9 +485,21 @@ func (c Config) Merge() Config {
 				}
 			}
 			if _, ok := cp[subSys]; !ok {
-				// A config subsystem was removed or server was downgraded.
-				Logger.Info("config: ignoring unknown subsystem config %q\n", subSys)
-				continue
+				rnSubSys, ok := renamedSubsys[subSys]
+				if !ok {
+					// A config subsystem was removed or server was downgraded.
+					Logger.Info("config: ignoring unknown subsystem config %q\n", subSys)
+					continue
+				}
+				// Copy over settings from previous sub-system
+				// to newly renamed sub-system
+				for _, kv := range cp[rnSubSys][Default] {
+					_, ok := c[subSys][tgt].Lookup(kv.Key)
+					if !ok {
+						ckvs.Set(kv.Key, kv.Value)
+					}
+				}
+				subSys = rnSubSys
 			}
 			cp[subSys][tgt] = ckvs
 		}

cmd/config/dns/etcd_dns.go

@@ -34,6 +34,9 @@ import (
 // ErrNoEntriesFound - Indicates no entries were found for the given key (directory)
 var ErrNoEntriesFound = errors.New("No entries found for this key")
 
+// ErrDomainMissing - Indicates domain is missing
+var ErrDomainMissing = errors.New("domain is missing")
+
 const etcdPathSeparator = "/"
 
 // create a new coredns service record for the bucket.
@@ -57,9 +60,9 @@ func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
 	var srvRecords = map[string][]SrvRecord{}
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.", domainName), c.prefixPath)
-		records, err := c.list(key)
+		records, err := c.list(key+etcdPathSeparator, true)
 		if err != nil {
-			return nil, err
+			return srvRecords, err
 		}
 		for _, record := range records {
 			if record.Key == "" {
@@ -76,7 +79,7 @@ func (c *CoreDNS) Get(bucket string) ([]SrvRecord, error) {
 	var srvRecords []SrvRecord
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
-		records, err := c.list(key)
+		records, err := c.list(key, false)
 		if err != nil {
 			return nil, err
 		}
@@ -109,19 +112,25 @@ func msgUnPath(s string) string {
 
 // Retrieves list of entries under the key passed.
 // Note that this method fetches entries upto only two levels deep.
-func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
+func (c *CoreDNS) list(key string, domain bool) ([]SrvRecord, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 	r, err := c.etcdClient.Get(ctx, key, clientv3.WithPrefix())
 	defer cancel()
 	if err != nil {
 		return nil, err
 	}
+
 	if r.Count == 0 {
 		key = strings.TrimSuffix(key, etcdPathSeparator)
 		r, err = c.etcdClient.Get(ctx, key)
 		if err != nil {
 			return nil, err
 		}
+		// only if we are looking at `domain` as true
+		// we should return error here.
+		if domain && r.Count == 0 {
+			return nil, ErrDomainMissing
+		}
 	}
 
 	var srvRecords []SrvRecord
@@ -166,11 +175,11 @@ func (c *CoreDNS) Put(bucket string) error {
 			key = key + etcdPathSeparator + ip
 			ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 			_, err = c.etcdClient.Put(ctx, key, string(bucketMsg))
-			defer cancel()
+			cancel()
 			if err != nil {
 				ctx, cancel = context.WithTimeout(context.Background(), defaultContextTimeout)
 				c.etcdClient.Delete(ctx, key)
-				defer cancel()
+				cancel()
 				return err
 			}
 		}
@@ -182,18 +191,12 @@ func (c *CoreDNS) Put(bucket string) error {
 func (c *CoreDNS) Delete(bucket string) error {
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
-		srvRecords, err := c.list(key)
+		ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+		_, err := c.etcdClient.Delete(ctx, key+etcdPathSeparator, clientv3.WithPrefix())
+		cancel()
 		if err != nil {
 			return err
 		}
-		for _, record := range srvRecords {
-			dctx, dcancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-			if _, err = c.etcdClient.Delete(dctx, key+etcdPathSeparator+record.Host); err != nil {
-				dcancel()
-				return err
-			}
-			dcancel()
-		}
 	}
 	return nil
 }
@@ -203,12 +206,12 @@ func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), c.prefixPath)
 
-		dctx, dcancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-		if _, err := c.etcdClient.Delete(dctx, key+etcdPathSeparator+record.Host); err != nil {
-			dcancel()
+		ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+		_, err := c.etcdClient.Delete(ctx, key+etcdPathSeparator+record.Host)
+		cancel()
+		if err != nil {
 			return err
 		}
-		dcancel()
 	}
 	return nil
 }

cmd/config/errors.go

@@ -30,6 +30,12 @@ var (
 		"Can only accept `on` and `off` values. To enable O_SYNC for fs backend, set this value to `on`",
 	)
 
+	ErrOverlappingDomainValue = newErrFn(
+		"Overlapping domain values",
+		"Please check the passed value",
+		"MINIO_DOMAIN only accepts non-overlapping domain values",
+	)
+
 	ErrInvalidDomainValue = newErrFn(
 		"Invalid domain value",
 		"Please check the passed value",
@@ -275,4 +281,10 @@ Example 1:
 		"",
 		"Refer to https://docs.min.io/docs/minio-kms-quickstart-guide.html for setting up SSE",
 	)
+
+	ErrInvalidReplicationWorkersValue = newErrFn(
+		"Invalid value for replication workers",
+		"",
+		"MINIO_API_REPLICATION_WORKERS: should be > 0",
+	)
 )

cmd/config/etcd/etcd.go

@@ -27,6 +27,7 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 	"go.etcd.io/etcd/clientv3"
 	"go.etcd.io/etcd/clientv3/namespace"
+	"go.uber.org/zap"
 )
 
 const (
@@ -144,6 +145,13 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
 	cfg.Enabled = true
 	cfg.DialTimeout = defaultDialTimeout
 	cfg.DialKeepAliveTime = defaultDialKeepAlive
+	// Disable etcd client SDK logging, etcd client
+	// incorrectly starts logging in unexpected data
+	// format.
+	cfg.LogConfig = &zap.Config{
+		Level:    zap.NewAtomicLevelAt(zap.FatalLevel),
+		Encoding: "console",
+	}
 	cfg.Endpoints = etcdEndpoints
 	cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kvs.Get(CoreDNSPath))
 	// Default path prefix for all keys on etcd, other than CoreDNSPath.

cmd/config/heal/heal.go

@@ -66,7 +66,7 @@ var (
 	Help = config.HelpKVS{
 		config.HelpKV{
 			Key:         Bitrot,
-			Description: `perform bitrot scan on disks when checking objects during crawl`,
+			Description: `perform bitrot scan on disks when checking objects during scanner`,
 			Optional:    true,
 			Type:        "on|off",
 		},

cmd/config/identity/ldap/config.go

@@ -21,17 +21,20 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"math/rand"
 	"net"
 	"strings"
 	"time"
 
+	ldap "github.com/go-ldap/ldap/v3"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
-	ldap "gopkg.in/ldap.v3"
 )
 
 const (
 	defaultLDAPExpiry = time.Hour * 1
+
+	dnDelimiter = ";"
 )
 
 // Config contains AD/LDAP server connectivity information.
@@ -45,50 +48,65 @@ type Config struct {
 	STSExpiryDuration string `json:"stsExpiryDuration"`
 
 	// Format string for usernames
-	UsernameFormat        string   `json:"usernameFormat"`
-	UsernameFormats       []string `json:"-"`
-	UsernameSearchFilter  string   `json:"-"`
-	UsernameSearchBaseDNS []string `json:"-"`
+	UsernameFormat  string   `json:"usernameFormat"`
+	UsernameFormats []string `json:"-"`
 
-	GroupSearchBaseDN  string   `json:"groupSearchBaseDN"`
-	GroupSearchBaseDNS []string `json:"-"`
-	GroupSearchFilter  string   `json:"groupSearchFilter"`
-	GroupNameAttribute string   `json:"groupNameAttribute"`
+	// User DN search parameters
+	UserDNSearchBaseDN string `json:"userDNSearchBaseDN"`
+	UserDNSearchFilter string `json:"userDNSearchFilter"`
+
+	// Group search parameters
+	GroupSearchBaseDistName  string   `json:"groupSearchBaseDN"`
+	GroupSearchBaseDistNames []string `json:"-"`
+	GroupSearchFilter        string   `json:"groupSearchFilter"`
+
+	// Lookup bind LDAP service account
+	LookupBindDN       string `json:"lookupBindDN"`
+	LookupBindPassword string `json:"lookupBindPassword"`
 
 	stsExpiryDuration time.Duration // contains converted value
 	tlsSkipVerify     bool          // allows skipping TLS verification
-	serverInsecure    bool          // allows plain text connection to LDAP Server
-	serverStartTLS    bool          // allows plain text connection to LDAP Server
+	serverInsecure    bool          // allows plain text connection to LDAP server
+	serverStartTLS    bool          // allows using StartTLS connection to LDAP server
+	isUsingLookupBind bool
 	rootCAs           *x509.CertPool
 }
 
 // LDAP keys and envs.
 const (
-	ServerAddr           = "server_addr"
-	STSExpiry            = "sts_expiry"
-	UsernameFormat       = "username_format"
-	UsernameSearchFilter = "username_search_filter"
-	UsernameSearchBaseDN = "username_search_base_dn"
-	GroupSearchFilter    = "group_search_filter"
-	GroupNameAttribute   = "group_name_attribute"
-	GroupSearchBaseDN    = "group_search_base_dn"
-	TLSSkipVerify        = "tls_skip_verify"
-	ServerInsecure       = "server_insecure"
-	ServerStartTLS       = "server_starttls"
+	ServerAddr         = "server_addr"
+	STSExpiry          = "sts_expiry"
+	LookupBindDN       = "lookup_bind_dn"
+	LookupBindPassword = "lookup_bind_password"
+	UserDNSearchBaseDN = "user_dn_search_base_dn"
+	UserDNSearchFilter = "user_dn_search_filter"
+	UsernameFormat     = "username_format"
+	GroupSearchFilter  = "group_search_filter"
+	GroupSearchBaseDN  = "group_search_base_dn"
+	TLSSkipVerify      = "tls_skip_verify"
+	ServerInsecure     = "server_insecure"
+	ServerStartTLS     = "server_starttls"
 
-	EnvServerAddr           = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
-	EnvSTSExpiry            = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
-	EnvTLSSkipVerify        = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"
-	EnvServerInsecure       = "MINIO_IDENTITY_LDAP_SERVER_INSECURE"
-	EnvServerStartTLS       = "MINIO_IDENTITY_LDAP_SERVER_STARTTLS"
-	EnvUsernameFormat       = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
-	EnvUsernameSearchFilter = "MINIO_IDENTITY_LDAP_USERNAME_SEARCH_FILTER"
-	EnvUsernameSearchBaseDN = "MINIO_IDENTITY_LDAP_USERNAME_SEARCH_BASE_DN"
-	EnvGroupSearchFilter    = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
-	EnvGroupNameAttribute   = "MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE"
-	EnvGroupSearchBaseDN    = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
+	EnvServerAddr         = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
+	EnvSTSExpiry          = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
+	EnvTLSSkipVerify      = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"
+	EnvServerInsecure     = "MINIO_IDENTITY_LDAP_SERVER_INSECURE"
+	EnvServerStartTLS     = "MINIO_IDENTITY_LDAP_SERVER_STARTTLS"
+	EnvUsernameFormat     = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
+	EnvUserDNSearchBaseDN = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN"
+	EnvUserDNSearchFilter = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER"
+	EnvGroupSearchFilter  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
+	EnvGroupSearchBaseDN  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
+	EnvLookupBindDN       = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN"
+	EnvLookupBindPassword = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD"
 )
 
+var removedKeys = []string{
+	"username_search_filter",
+	"username_search_base_dn",
+	"group_name_attribute",
+}
+
 // DefaultKVS - default config for LDAP config
 var (
 	DefaultKVS = config.KVS{
@@ -101,21 +119,17 @@ var (
 			Value: "",
 		},
 		config.KV{
-			Key:   UsernameSearchFilter,
+			Key:   UserDNSearchBaseDN,
 			Value: "",
 		},
 		config.KV{
-			Key:   UsernameSearchBaseDN,
+			Key:   UserDNSearchFilter,
 			Value: "",
 		},
 		config.KV{
 			Key:   GroupSearchFilter,
 			Value: "",
 		},
-		config.KV{
-			Key:   GroupNameAttribute,
-			Value: "",
-		},
 		config.KV{
 			Key:   GroupSearchBaseDN,
 			Value: "",
@@ -136,117 +150,191 @@ var (
 			Key:   ServerStartTLS,
 			Value: config.EnableOff,
 		},
+		config.KV{
+			Key:   LookupBindDN,
+			Value: "",
+		},
+		config.KV{
+			Key:   LookupBindPassword,
+			Value: "",
+		},
 	}
 )
 
-const (
-	dnDelimiter = ";"
-)
-
 func getGroups(conn *ldap.Conn, sreq *ldap.SearchRequest) ([]string, error) {
 	var groups []string
 	sres, err := conn.Search(sreq)
 	if err != nil {
+		// Check if there is no matching result and return empty slice.
+		// Ref: https://ldap.com/ldap-result-code-reference/
+		if ldap.IsErrorWithCode(err, 32) {
+			return nil, nil
+		}
 		return nil, err
 	}
 	for _, entry := range sres.Entries {
 		// We only queried one attribute,
 		// so we only look up the first one.
-		groups = append(groups, entry.Attributes[0].Values...)
+		groups = append(groups, entry.DN)
 	}
 	return groups, nil
 }
 
-func (l *Config) bind(conn *ldap.Conn, username, password string) ([]string, error) {
-	var bindDNS = make([]string, len(l.UsernameFormats))
+func (l *Config) lookupBind(conn *ldap.Conn) error {
+	var err error
+	if l.LookupBindPassword == "" {
+		err = conn.UnauthenticatedBind(l.LookupBindDN)
+	} else {
+		err = conn.Bind(l.LookupBindDN, l.LookupBindPassword)
+	}
+	if ldap.IsErrorWithCode(err, 49) {
+		return fmt.Errorf("LDAP Lookup Bind user invalid credentials error: %v", err)
+	}
+	return err
+}
+
+// usernameFormatsBind - Iterates over all given username formats and expects
+// that only one will succeed if the credentials are valid. The succeeding
+// bindDN is returned or an error.
+//
+// In the rare case that multiple username formats succeed, implying that two
+// (or more) distinct users in the LDAP directory have the same username and
+// password, we return an error as we cannot identify the account intended by
+// the user.
+func (l *Config) usernameFormatsBind(conn *ldap.Conn, username, password string) (string, error) {
+	var bindDistNames []string
+	var errs = make([]error, len(l.UsernameFormats))
+	var successCount = 0
 	for i, usernameFormat := range l.UsernameFormats {
 		bindDN := fmt.Sprintf(usernameFormat, username)
 		// Bind with user credentials to validate the password
-		if err := conn.Bind(bindDN, password); err != nil {
-			return nil, err
+		errs[i] = conn.Bind(bindDN, password)
+		if errs[i] == nil {
+			bindDistNames = append(bindDistNames, bindDN)
+			successCount++
+		} else if !ldap.IsErrorWithCode(errs[i], 49) {
+			return "", fmt.Errorf("LDAP Bind request failed with unexpected error: %v", errs[i])
 		}
-		bindDNS[i] = bindDN
 	}
-	return bindDNS, nil
+	if successCount == 0 {
+		var errStrings []string
+		for _, err := range errs {
+			if err != nil {
+				errStrings = append(errStrings, err.Error())
+			}
+		}
+		outErr := fmt.Sprintf("All username formats failed due to invalid credentials: %s", strings.Join(errStrings, "; "))
+		return "", errors.New(outErr)
+	}
+	if successCount > 1 {
+		successDistNames := strings.Join(bindDistNames, ", ")
+		errMsg := fmt.Sprintf("Multiple username formats succeeded - ambiguous user login (succeeded for: %s)", successDistNames)
+		return "", errors.New(errMsg)
+	}
+	return bindDistNames[0], nil
 }
 
-var standardAttributes = []string{
-	"givenName",
-	"sn",
-	"cn",
-	"memberOf",
-	"email",
+// lookupUserDN searches for the DN of the user given their username. conn is
+// assumed to be using the lookup bind service account. It is required that the
+// search result in at most one result.
+func (l *Config) lookupUserDN(conn *ldap.Conn, username string) (string, error) {
+	filter := strings.Replace(l.UserDNSearchFilter, "%s", ldap.EscapeFilter(username), -1)
+	searchRequest := ldap.NewSearchRequest(
+		l.UserDNSearchBaseDN,
+		ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+		filter,
+		[]string{}, // only need DN, so no pass no attributes here
+		nil,
+	)
+
+	searchResult, err := conn.Search(searchRequest)
+	if err != nil {
+		return "", err
+	}
+	if len(searchResult.Entries) == 0 {
+		return "", fmt.Errorf("User DN for %s not found", username)
+	}
+	if len(searchResult.Entries) != 1 {
+		return "", fmt.Errorf("Multiple DNs for %s found - please fix the search filter", username)
+	}
+	return searchResult.Entries[0].DN, nil
 }
 
-// Bind - binds to ldap, searches LDAP and returns list of groups.
-func (l *Config) Bind(username, password string) ([]string, error) {
+// Bind - binds to ldap, searches LDAP and returns the distinguished name of the
+// user and the list of groups.
+func (l *Config) Bind(username, password string) (string, []string, error) {
 	conn, err := l.Connect()
 	if err != nil {
-		return nil, err
+		return "", nil, err
 	}
 	defer conn.Close()
 
-	bindDNS, err := l.bind(conn, username, password)
-	if err != nil {
-		return nil, err
+	var bindDN string
+	if l.isUsingLookupBind {
+		// Bind to the lookup user account
+		if err = l.lookupBind(conn); err != nil {
+			return "", nil, err
+		}
+
+		// Lookup user DN
+		bindDN, err = l.lookupUserDN(conn, username)
+		if err != nil {
+			errRet := fmt.Errorf("Unable to find user DN: %s", err)
+			return "", nil, errRet
+		}
+
+		// Authenticate the user credentials.
+		err = conn.Bind(bindDN, password)
+		if err != nil {
+			errRet := fmt.Errorf("LDAP auth failed for DN %s: %v", bindDN, err)
+			return "", nil, errRet
+		}
+
+		// Bind to the lookup user account again to perform group search.
+		if err = l.lookupBind(conn); err != nil {
+			return "", nil, err
+		}
+	} else {
+		// Verify login credentials by checking the username formats.
+		bindDN, err = l.usernameFormatsBind(conn, username, password)
+		if err != nil {
+			return "", nil, err
+		}
+
+		// Bind to the successful bindDN again.
+		err = conn.Bind(bindDN, password)
+		if err != nil {
+			errRet := fmt.Errorf("LDAP conn failed though auth for DN %s succeeded: %v", bindDN, err)
+			return "", nil, errRet
+		}
 	}
 
+	// User groups lookup.
 	var groups []string
-	if l.UsernameSearchFilter != "" {
-		for _, userSearchBase := range l.UsernameSearchBaseDNS {
-			filter := strings.Replace(l.UsernameSearchFilter, "%s",
-				ldap.EscapeFilter(username), -1)
-
+	if l.GroupSearchFilter != "" {
+		for _, groupSearchBase := range l.GroupSearchBaseDistNames {
+			filter := strings.Replace(l.GroupSearchFilter, "%s", ldap.EscapeFilter(username), -1)
+			filter = strings.Replace(filter, "%d", ldap.EscapeFilter(bindDN), -1)
 			searchRequest := ldap.NewSearchRequest(
-				userSearchBase,
+				groupSearchBase,
 				ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
 				filter,
-				standardAttributes,
+				nil,
 				nil,
 			)
 
-			groups, err = getGroups(conn, searchRequest)
+			var newGroups []string
+			newGroups, err = getGroups(conn, searchRequest)
 			if err != nil {
-				return nil, err
+				errRet := fmt.Errorf("Error finding groups of %s: %v", bindDN, err)
+				return "", nil, errRet
 			}
+
+			groups = append(groups, newGroups...)
 		}
 	}
 
-	if l.GroupSearchFilter != "" {
-		for _, groupSearchBase := range l.GroupSearchBaseDNS {
-			var filters []string
-			if l.GroupNameAttribute == "" {
-				filters = []string{strings.Replace(l.GroupSearchFilter, "%s",
-					ldap.EscapeFilter(username), -1)}
-			} else {
-				// With group name attribute specified, make sure to
-				// include search queries for CN distinguished name
-				for _, bindDN := range bindDNS {
-					filters = append(filters, strings.Replace(l.GroupSearchFilter, "%s",
-						ldap.EscapeFilter(bindDN), -1))
-				}
-			}
-			for _, filter := range filters {
-				searchRequest := ldap.NewSearchRequest(
-					groupSearchBase,
-					ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
-					filter,
-					standardAttributes,
-					nil,
-				)
-
-				var newGroups []string
-				newGroups, err = getGroups(conn, searchRequest)
-				if err != nil {
-					return nil, err
-				}
-
-				groups = append(groups, newGroups...)
-			}
-		}
-	}
-
-	return groups, nil
+	return bindDN, groups, nil
 }
 
 // Connect connect to ldap server.
@@ -287,6 +375,38 @@ func (l Config) GetExpiryDuration() time.Duration {
 	return l.stsExpiryDuration
 }
 
+func (l Config) testConnection() error {
+	conn, err := l.Connect()
+	if err != nil {
+		return fmt.Errorf("Error creating connection to LDAP server: %v", err)
+	}
+	defer conn.Close()
+	if l.isUsingLookupBind {
+		if err = l.lookupBind(conn); err != nil {
+			return fmt.Errorf("Error connecting as LDAP Lookup Bind user: %v", err)
+		}
+		return nil
+	}
+
+	// Generate some random user credentials for username formats mode test.
+	username := fmt.Sprintf("sometestuser%09d", rand.Int31n(1000000000))
+	charset := []byte("abcdefghijklmnopqrstuvwxyz" +
+		"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+	rand.Shuffle(len(charset), func(i, j int) {
+		charset[i], charset[j] = charset[j], charset[i]
+	})
+	password := string(charset[:20])
+	_, err = l.usernameFormatsBind(conn, username, password)
+	if err == nil {
+		// We don't expect to successfully guess a credential in this
+		// way.
+		return fmt.Errorf("Unexpected random credentials success for user=%s password=%s", username, password)
+	} else if strings.HasPrefix(err.Error(), "All username formats failed due to invalid credentials: ") {
+		return nil
+	}
+	return fmt.Errorf("LDAP connection test error: %v", err)
+}
+
 // Enabled returns if jwks is enabled.
 func Enabled(kvs config.KVS) bool {
 	return kvs.Get(ServerAddr) != ""
@@ -295,6 +415,12 @@ func Enabled(kvs config.KVS) bool {
 // Lookup - initializes LDAP config, overrides config, if any ENV values are set.
 func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 	l = Config{}
+
+	// Purge all removed keys first
+	for _, k := range removedKeys {
+		kvs.Delete(k)
+	}
+
 	if err = config.CheckValidKeys(config.IdentityLDAPSubSys, kvs, DefaultKVS); err != nil {
 		return l, err
 	}
@@ -316,6 +442,8 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 		l.STSExpiryDuration = v
 		l.stsExpiryDuration = expDur
 	}
+
+	// LDAP connection configuration
 	if v := env.Get(EnvServerInsecure, kvs.Get(ServerInsecure)); v != "" {
 		l.serverInsecure, err = config.ParseBool(v)
 		if err != nil {
@@ -334,43 +462,62 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 			return l, err
 		}
 	}
+
+	// Lookup bind user configuration
+	lookupBindDN := env.Get(EnvLookupBindDN, kvs.Get(LookupBindDN))
+	lookupBindPassword := env.Get(EnvLookupBindPassword, kvs.Get(LookupBindPassword))
+	if lookupBindDN != "" {
+		l.LookupBindDN = lookupBindDN
+		l.LookupBindPassword = lookupBindPassword
+		l.isUsingLookupBind = true
+
+		// User DN search configuration
+		userDNSearchBaseDN := env.Get(EnvUserDNSearchBaseDN, kvs.Get(UserDNSearchBaseDN))
+		userDNSearchFilter := env.Get(EnvUserDNSearchFilter, kvs.Get(UserDNSearchFilter))
+		if userDNSearchFilter == "" || userDNSearchBaseDN == "" {
+			return l, errors.New("In lookup bind mode, userDN search base DN and userDN search filter are both required")
+		}
+		l.UserDNSearchBaseDN = userDNSearchBaseDN
+		l.UserDNSearchFilter = userDNSearchFilter
+	}
+
+	// Username format configuration.
 	if v := env.Get(EnvUsernameFormat, kvs.Get(UsernameFormat)); v != "" {
 		if !strings.Contains(v, "%s") {
 			return l, errors.New("LDAP username format doesn't have '%s' substitution")
 		}
 		l.UsernameFormats = strings.Split(v, dnDelimiter)
-	} else {
-		return l, fmt.Errorf("'%s' cannot be empty and must have a value", UsernameFormat)
 	}
 
-	if v := env.Get(EnvUsernameSearchFilter, kvs.Get(UsernameSearchFilter)); v != "" {
-		if !strings.Contains(v, "%s") {
-			return l, errors.New("LDAP username search filter doesn't have '%s' substitution")
-		}
-		l.UsernameSearchFilter = v
+	// Either lookup bind mode or username format is supported, but not
+	// both.
+	if l.isUsingLookupBind && len(l.UsernameFormats) > 0 {
+		return l, errors.New("Lookup Bind mode and Username Format mode are not supported at the same time")
 	}
 
-	if v := env.Get(EnvUsernameSearchBaseDN, kvs.Get(UsernameSearchBaseDN)); v != "" {
-		l.UsernameSearchBaseDNS = strings.Split(v, dnDelimiter)
+	// At least one of bind mode or username format must be used.
+	if !l.isUsingLookupBind && len(l.UsernameFormats) == 0 {
+		return l, errors.New("Either Lookup Bind mode or Username Format mode is required.")
 	}
 
+	// Test connection to LDAP server.
+	if err := l.testConnection(); err != nil {
+		return l, fmt.Errorf("Connection test for LDAP server failed: %v", err)
+	}
+
+	// Group search params configuration
 	grpSearchFilter := env.Get(EnvGroupSearchFilter, kvs.Get(GroupSearchFilter))
-	grpSearchNameAttr := env.Get(EnvGroupNameAttribute, kvs.Get(GroupNameAttribute))
 	grpSearchBaseDN := env.Get(EnvGroupSearchBaseDN, kvs.Get(GroupSearchBaseDN))
 
 	// Either all group params must be set or none must be set.
-	var allSet bool
-	if grpSearchFilter != "" {
-		if grpSearchNameAttr == "" || grpSearchBaseDN == "" {
-			return l, errors.New("All group related parameters must be set")
-		}
-		allSet = true
+	if (grpSearchFilter != "" && grpSearchBaseDN == "") || (grpSearchFilter == "" && grpSearchBaseDN != "") {
+		return l, errors.New("All group related parameters must be set")
 	}
 
-	if allSet {
+	if grpSearchFilter != "" {
 		l.GroupSearchFilter = grpSearchFilter
-		l.GroupNameAttribute = grpSearchNameAttr
-		l.GroupSearchBaseDNS = strings.Split(grpSearchBaseDN, dnDelimiter)
+		l.GroupSearchBaseDistName = grpSearchBaseDN
+		l.GroupSearchBaseDistNames = strings.Split(l.GroupSearchBaseDistName, dnDelimiter)
 	}
 
 	l.rootCAs = rootCAs

cmd/config/identity/ldap/help.go

@@ -27,43 +27,53 @@ var (
 			Type:        "address",
 		},
 		config.HelpKV{
-			Key:         UsernameFormat,
-			Description: `";" separated list of username bind DNs e.g. "uid=%s,cn=accounts,dc=myldapserver,dc=com"`,
-			Type:        "list",
+			Key:         STSExpiry,
+			Description: `temporary credentials validity duration in s,m,h,d. Default is "1h"`,
+			Optional:    true,
+			Type:        "duration",
 		},
 		config.HelpKV{
-			Key:         UsernameSearchFilter,
-			Description: `user search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)"`,
+			Key:         LookupBindDN,
+			Description: `DN for LDAP read-only service account used to perform DN and group lookups`,
+			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         LookupBindPassword,
+			Description: `Password for LDAP read-only service account used to perform DN and group lookups`,
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         UserDNSearchBaseDN,
+			Description: `Base LDAP DN to search for user DN`,
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         UserDNSearchFilter,
+			Description: `Search filter to lookup user DN`,
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         UsernameFormat,
+			Description: `";" separated list of username bind DNs e.g. "uid=%s,cn=accounts,dc=myldapserver,dc=com"`,
+			Optional:    true,
+			Type:        "list",
+		},
 		config.HelpKV{
 			Key:         GroupSearchFilter,
 			Description: `search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"`,
+			Optional:    true,
 			Type:        "string",
 		},
 		config.HelpKV{
 			Key:         GroupSearchBaseDN,
 			Description: `";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"`,
+			Optional:    true,
 			Type:        "list",
 		},
-		config.HelpKV{
-			Key:         UsernameSearchBaseDN,
-			Description: `";" separated list of username search DNs`,
-			Type:        "list",
-			Optional:    true,
-		},
-		config.HelpKV{
-			Key:         GroupNameAttribute,
-			Description: `search attribute for group name e.g. "cn"`,
-			Optional:    true,
-			Type:        "string",
-		},
-		config.HelpKV{
-			Key:         STSExpiry,
-			Description: `temporary credentials validity duration in s,m,h,d. Default is "1h"`,
-			Optional:    true,
-			Type:        "duration",
-		},
 		config.HelpKV{
 			Key:         TLSSkipVerify,
 			Description: `trust server TLS without verification, defaults to "off" (verify)`,
@@ -76,6 +86,12 @@ var (
 			Optional:    true,
 			Type:        "on|off",
 		},
+		config.HelpKV{
+			Key:         ServerStartTLS,
+			Description: `use StartTLS connection to AD/LDAP server, defaults to "off"`,
+			Optional:    true,
+			Type:        "on|off",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

cmd/config/identity/ldap/legacy.go

@@ -41,13 +41,9 @@ func SetIdentityLDAP(s config.Config, ldapArgs Config) {
 			Key:   GroupSearchFilter,
 			Value: ldapArgs.GroupSearchFilter,
 		},
-		config.KV{
-			Key:   GroupNameAttribute,
-			Value: ldapArgs.GroupNameAttribute,
-		},
 		config.KV{
 			Key:   GroupSearchBaseDN,
-			Value: ldapArgs.GroupSearchBaseDN,
+			Value: ldapArgs.GroupSearchBaseDistName,
 		},
 	}
 }

cmd/config/identity/openid/ecdsa-sha3.go

@@ -1,18 +1,18 @@
-/*
- * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !fips
 
 package openid
 
@@ -22,7 +22,7 @@ import (
 	"github.com/dgrijalva/jwt-go"
 
 	// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
-	_ "golang.org/x/crypto/sha3"
+	_ "golang.org/x/crypto/sha3" // There is no SHA-3 FIPS-140 2 compliant implementation
 )
 
 // Specific instances for EC256 and company

cmd/config/identity/openid/rsa-sha3.go

@@ -1,18 +1,18 @@
-/*
- * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !fips
 
 package openid
 
@@ -22,7 +22,7 @@ import (
 	"github.com/dgrijalva/jwt-go"
 
 	// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
-	_ "golang.org/x/crypto/sha3"
+	_ "golang.org/x/crypto/sha3" // There is no SHA-3 FIPS-140 2 compliant implementation
 )
 
 // Specific instances for RS256 and company

cmd/config/scanner/scanner.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package crawler
+package scanner
 
 import (
 	"strconv"
@@ -29,8 +29,10 @@ const (
 	Delay   = "delay"
 	MaxWait = "max_wait"
 
-	EnvDelay   = "MINIO_CRAWLER_DELAY"
-	EnvMaxWait = "MINIO_CRAWLER_MAX_WAIT"
+	EnvDelay         = "MINIO_SCANNER_DELAY"
+	EnvDelayLegacy   = "MINIO_CRAWLER_DELAY"
+	EnvMaxWait       = "MINIO_SCANNER_MAX_WAIT"
+	EnvMaxWaitLegacy = "MINIO_CRAWLER_MAX_WAIT"
 )
 
 // Config represents the heal settings.
@@ -58,7 +60,7 @@ var (
 	Help = config.HelpKVS{
 		config.HelpKV{
 			Key:         Delay,
-			Description: `crawler delay multiplier, defaults to '10.0'`,
+			Description: `scanner delay multiplier, defaults to '10.0'`,
 			Optional:    true,
 			Type:        "float",
 		},
@@ -73,14 +75,22 @@ var (
 
 // LookupConfig - lookup config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS) (cfg Config, err error) {
-	if err = config.CheckValidKeys(config.CrawlerSubSys, kvs, DefaultKVS); err != nil {
+	if err = config.CheckValidKeys(config.ScannerSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
-	cfg.Delay, err = strconv.ParseFloat(env.Get(EnvDelay, kvs.Get(Delay)), 64)
+	delay := env.Get(EnvDelayLegacy, "")
+	if delay == "" {
+		delay = env.Get(EnvDelay, kvs.Get(Delay))
+	}
+	cfg.Delay, err = strconv.ParseFloat(delay, 64)
 	if err != nil {
 		return cfg, err
 	}
-	cfg.MaxWait, err = time.ParseDuration(env.Get(EnvMaxWait, kvs.Get(MaxWait)))
+	maxWait := env.Get(EnvMaxWaitLegacy, "")
+	if maxWait == "" {
+		maxWait = env.Get(EnvMaxWait, kvs.Get(MaxWait))
+	}
+	cfg.MaxWait, err = time.ParseDuration(maxWait)
 	if err != nil {
 		return cfg, err
 	}

cmd/config/storageclass/storage-class.go

@@ -64,7 +64,7 @@ const (
 	defaultRRSParity = minParityDisks
 
 	// Default DMA value
-	defaultDMA = DMAWrite
+	defaultDMA = DMAReadWrite
 )
 
 // DefaultKVS - default storage class config
@@ -88,14 +88,13 @@ var (
 // StorageClass - holds storage class information
 type StorageClass struct {
 	Parity int
-	DMA    string
 }
 
 // Config storage class configuration
 type Config struct {
 	Standard StorageClass `json:"standard"`
 	RRS      StorageClass `json:"rrs"`
-	DMA      StorageClass `json:"dma"`
+	DMA      string       `json:"dma"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -112,7 +111,7 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 // IsValid - returns true if input string is a valid
 // storage class kind supported.
 func IsValid(sc string) bool {
-	return sc == RRS || sc == STANDARD || sc == DMA
+	return sc == RRS || sc == STANDARD
 }
 
 // UnmarshalText unmarshals storage class from its textual form into
@@ -122,14 +121,6 @@ func (sc *StorageClass) UnmarshalText(b []byte) error {
 	if scStr == "" {
 		return nil
 	}
-	if scStr == DMAWrite {
-		sc.DMA = DMAWrite
-		return nil
-	}
-	if scStr == DMAReadWrite {
-		sc.DMA = DMAReadWrite
-		return nil
-	}
 	s, err := parseStorageClass(scStr)
 	if err != nil {
 		return err
@@ -143,14 +134,14 @@ func (sc *StorageClass) MarshalText() ([]byte, error) {
 	if sc.Parity != 0 {
 		return []byte(fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)), nil
 	}
-	return []byte(sc.DMA), nil
+	return []byte{}, nil
 }
 
 func (sc *StorageClass) String() string {
 	if sc.Parity != 0 {
 		return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
 	}
-	return sc.DMA
+	return ""
 }
 
 // Parses given storageClassEnv and returns a storageClass structure.
@@ -182,22 +173,34 @@ func parseStorageClass(storageClassEnv string) (sc StorageClass, err error) {
 	}, nil
 }
 
-// Validates the parity disks.
-func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
-	if ssParity == 0 && rrsParity == 0 {
-		return nil
-	}
-
+// ValidateParity validate standard storage class parity.
+func ValidateParity(ssParity, setDriveCount int) error {
 	// SS parity disks should be greater than or equal to minParityDisks.
 	// Parity below minParityDisks is not supported.
-	if ssParity < minParityDisks {
+	if ssParity > 0 && ssParity < minParityDisks {
+		return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d",
+			ssParity, minParityDisks)
+	}
+
+	if ssParity > setDriveCount/2 {
+		return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, setDriveCount/2)
+	}
+
+	return nil
+}
+
+// Validates the parity disks.
+func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
+	// SS parity disks should be greater than or equal to minParityDisks.
+	// Parity below minParityDisks is not supported.
+	if ssParity > 0 && ssParity < minParityDisks {
 		return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d",
 			ssParity, minParityDisks)
 	}
 
 	// RRS parity disks should be greater than or equal to minParityDisks.
 	// Parity below minParityDisks is not supported.
-	if rrsParity < minParityDisks {
+	if rrsParity > 0 && rrsParity < minParityDisks {
 		return fmt.Errorf("Reduced redundancy storage class parity %d should be greater than or equal to %d", rrsParity, minParityDisks)
 	}
 
@@ -210,7 +213,7 @@ func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
 	}
 
 	if ssParity > 0 && rrsParity > 0 {
-		if ssParity < rrsParity {
+		if ssParity > 0 && ssParity < rrsParity {
 			return fmt.Errorf("Standard storage class parity disks %d should be greater than or equal to Reduced redundancy storage class parity disks %d", ssParity, rrsParity)
 		}
 	}
@@ -218,14 +221,16 @@ func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
 }
 
 // GetParityForSC - Returns the data and parity drive count based on storage class
-// If storage class is set using the env vars MINIO_STORAGE_CLASS_RRS and MINIO_STORAGE_CLASS_STANDARD
-// or config.json fields
-// -- corresponding values are returned
-// If storage class is not set during startup, default values are returned
-// -- Default for Reduced Redundancy Storage class is, parity = 2 and data = N-Parity
-// -- Default for Standard Storage class is, parity = N/2, data = N/2
-// If storage class is empty
-// -- standard storage class is assumed and corresponding data and parity is returned
+// If storage class is set using the env vars MINIO_STORAGE_CLASS_RRS and
+// MINIO_STORAGE_CLASS_STANDARD or server config fields corresponding values are
+// returned.
+//
+// -- if input storage class is empty then standard is assumed
+// -- if input is RRS but RRS is not configured default '2' parity
+//    for RRS is assumed
+// -- if input is STANDARD but STANDARD is not configured '0' parity
+//    is returned, the caller is expected to choose the right parity
+//    at that point.
 func (sCfg Config) GetParityForSC(sc string) (parity int) {
 	switch strings.TrimSpace(sc) {
 	case RRS:
@@ -241,7 +246,7 @@ func (sCfg Config) GetParityForSC(sc string) (parity int) {
 
 // GetDMA - returns DMA configuration.
 func (sCfg Config) GetDMA() string {
-	return sCfg.DMA.DMA
+	return sCfg.DMA
 }
 
 // Enabled returns if etcd is enabled.
@@ -254,8 +259,6 @@ func Enabled(kvs config.KVS) bool {
 // LookupConfig - lookup storage class config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 	cfg = Config{}
-	cfg.Standard.Parity = setDriveCount / 2
-	cfg.RRS.Parity = defaultRRSParity
 
 	if err = config.CheckValidKeys(config.StorageClassSubSys, kvs, DefaultKVS); err != nil {
 		return Config{}, err
@@ -271,9 +274,6 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 			return Config{}, err
 		}
 	}
-	if cfg.Standard.Parity == 0 {
-		cfg.Standard.Parity = setDriveCount / 2
-	}
 
 	if rrsc != "" {
 		cfg.RRS, err = parseStorageClass(rrsc)
@@ -291,7 +291,7 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 	if dma != DMAReadWrite && dma != DMAWrite {
 		return Config{}, errors.New(`valid dma values are "read-write" and "write"`)
 	}
-	cfg.DMA.DMA = dma
+	cfg.DMA = dma
 
 	// Validation is done after parsing both the storage classes. This is needed because we need one
 	// storage class value to deduce the correct value of the other storage class.

cmd/crypto/config.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"math/rand"
 	"net/http"
+	"os"
 	"reflect"
 	"strconv"
 	"strings"
@@ -410,6 +411,9 @@ func NewKMS(cfg KMSConfig) (kms KMS, err error) {
 	} else if cfg.Vault.Enabled && cfg.Kes.Enabled {
 		return kms, errors.New("Ambiguous KMS configuration: vault configuration and kes configuration are provided at the same time")
 	} else if cfg.Vault.Enabled {
+		if v, ok := os.LookupEnv("MINIO_KMS_VAULT_DEPRECATION"); !ok || v != "off" { // TODO(aead): Remove once Vault support has been removed
+			return kms, errors.New("Hashicorp Vault is deprecated and will be removed Oct. 2021. To temporarily enable Hashicorp Vault support, set MINIO_KMS_VAULT_DEPRECATION=off")
+		}
 		kms, err = NewVault(cfg.Vault)
 		if err != nil {
 			return kms, err

cmd/crypto/json.go

@@ -0,0 +1,200 @@
+// MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"unicode/utf8"
+)
+
+// Adapted from Go stdlib.
+
+var hexTable = "0123456789abcdef"
+
+// EscapeStringJSON will escape a string for JSON and write it to dst.
+func EscapeStringJSON(dst *bytes.Buffer, s string) {
+	start := 0
+	for i := 0; i < len(s); {
+		if b := s[i]; b < utf8.RuneSelf {
+			if htmlSafeSet[b] {
+				i++
+				continue
+			}
+			if start < i {
+				dst.WriteString(s[start:i])
+			}
+			dst.WriteByte('\\')
+			switch b {
+			case '\\', '"':
+				dst.WriteByte(b)
+			case '\n':
+				dst.WriteByte('n')
+			case '\r':
+				dst.WriteByte('r')
+			case '\t':
+				dst.WriteByte('t')
+			default:
+				// This encodes bytes < 0x20 except for \t, \n and \r.
+				// If escapeHTML is set, it also escapes <, >, and &
+				// because they can lead to security holes when
+				// user-controlled strings are rendered into JSON
+				// and served to some browsers.
+				dst.WriteString(`u00`)
+				dst.WriteByte(hexTable[b>>4])
+				dst.WriteByte(hexTable[b&0xF])
+			}
+			i++
+			start = i
+			continue
+		}
+		c, size := utf8.DecodeRuneInString(s[i:])
+		if c == utf8.RuneError && size == 1 {
+			if start < i {
+				dst.WriteString(s[start:i])
+			}
+			dst.WriteString(`\ufffd`)
+			i += size
+			start = i
+			continue
+		}
+		// U+2028 is LINE SEPARATOR.
+		// U+2029 is PARAGRAPH SEPARATOR.
+		// They are both technically valid characters in JSON strings,
+		// but don't work in JSONP, which has to be evaluated as JavaScript,
+		// and can lead to security holes there. It is valid JSON to
+		// escape them, so we do so unconditionally.
+		// See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion.
+		if c == '\u2028' || c == '\u2029' {
+			if start < i {
+				dst.WriteString(s[start:i])
+			}
+			dst.WriteString(`\u202`)
+			dst.WriteByte(hexTable[c&0xF])
+			i += size
+			start = i
+			continue
+		}
+		i += size
+	}
+	if start < len(s) {
+		dst.WriteString(s[start:])
+	}
+}
+
+// htmlSafeSet holds the value true if the ASCII character with the given
+// array position can be safely represented inside a JSON string, embedded
+// inside of HTML <script> tags, without any additional escaping.
+//
+// All values are true except for the ASCII control characters (0-31), the
+// double quote ("), the backslash character ("\"), HTML opening and closing
+// tags ("<" and ">"), and the ampersand ("&").
+var htmlSafeSet = [utf8.RuneSelf]bool{
+	' ':      true,
+	'!':      true,
+	'"':      false,
+	'#':      true,
+	'$':      true,
+	'%':      true,
+	'&':      false,
+	'\'':     true,
+	'(':      true,
+	')':      true,
+	'*':      true,
+	'+':      true,
+	',':      true,
+	'-':      true,
+	'.':      true,
+	'/':      true,
+	'0':      true,
+	'1':      true,
+	'2':      true,
+	'3':      true,
+	'4':      true,
+	'5':      true,
+	'6':      true,
+	'7':      true,
+	'8':      true,
+	'9':      true,
+	':':      true,
+	';':      true,
+	'<':      false,
+	'=':      true,
+	'>':      false,
+	'?':      true,
+	'@':      true,
+	'A':      true,
+	'B':      true,
+	'C':      true,
+	'D':      true,
+	'E':      true,
+	'F':      true,
+	'G':      true,
+	'H':      true,
+	'I':      true,
+	'J':      true,
+	'K':      true,
+	'L':      true,
+	'M':      true,
+	'N':      true,
+	'O':      true,
+	'P':      true,
+	'Q':      true,
+	'R':      true,
+	'S':      true,
+	'T':      true,
+	'U':      true,
+	'V':      true,
+	'W':      true,
+	'X':      true,
+	'Y':      true,
+	'Z':      true,
+	'[':      true,
+	'\\':     false,
+	']':      true,
+	'^':      true,
+	'_':      true,
+	'`':      true,
+	'a':      true,
+	'b':      true,
+	'c':      true,
+	'd':      true,
+	'e':      true,
+	'f':      true,
+	'g':      true,
+	'h':      true,
+	'i':      true,
+	'j':      true,
+	'k':      true,
+	'l':      true,
+	'm':      true,
+	'n':      true,
+	'o':      true,
+	'p':      true,
+	'q':      true,
+	'r':      true,
+	's':      true,
+	't':      true,
+	'u':      true,
+	'v':      true,
+	'w':      true,
+	'x':      true,
+	'y':      true,
+	'z':      true,
+	'{':      true,
+	'|':      true,
+	'}':      true,
+	'~':      true,
+	'\u007f': true,
+}

cmd/crypto/kes.go

@@ -116,9 +116,9 @@ func NewKes(cfg KesConfig) (KMS, error) {
 	if err != nil {
 		return nil, err
 	}
-	if cfg.Transport.TLSClientConfig != nil {
-		if err = loadCACertificates(cfg.CAPath,
-			cfg.Transport.TLSClientConfig.RootCAs); err != nil {
+
+	if cfg.Transport.TLSClientConfig != nil && cfg.Transport.TLSClientConfig.RootCAs != nil {
+		if err = loadCACertificates(cfg.CAPath, cfg.Transport.TLSClientConfig.RootCAs); err != nil {
 			return nil, err
 		}
 	} else {
@@ -132,8 +132,12 @@ func NewKes(cfg KesConfig) (KMS, error) {
 		if err = loadCACertificates(cfg.CAPath, rootCAs); err != nil {
 			return nil, err
 		}
-		cfg.Transport.TLSClientConfig = &tls.Config{
-			RootCAs: rootCAs,
+		if cfg.Transport.TLSClientConfig == nil {
+			cfg.Transport.TLSClientConfig = &tls.Config{
+				RootCAs: rootCAs,
+			}
+		} else {
+			cfg.Transport.TLSClientConfig.RootCAs = rootCAs
 		}
 	}
 	cfg.Transport.TLSClientConfig.Certificates = []tls.Certificate{cert}

cmd/crypto/key.go

@@ -19,13 +19,13 @@ import (
 	"context"
 	"crypto/hmac"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/binary"
 	"errors"
 	"io"
 	"path"
 
 	"github.com/minio/minio/cmd/logger"
-	sha256 "github.com/minio/sha256-simd"
 	"github.com/minio/sio"
 )
 

cmd/crypto/kms.go

@@ -19,13 +19,13 @@ import (
 	"context"
 	"crypto/hmac"
 	"crypto/rand"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io"
 	"sort"
 
 	"github.com/minio/minio/cmd/logger"
-	sha256 "github.com/minio/sha256-simd"
 	"github.com/minio/sio"
 )
 
@@ -39,6 +39,7 @@ type Context map[string]string
 //
 // WriteTo sorts the context keys and writes the sorted
 // key-value pairs as canonical JSON object to w.
+// Sort order is based on the un-escaped keys.
 //
 // Note that neither keys nor values are escaped for JSON.
 func (c Context) WriteTo(w io.Writer) (n int64, err error) {
@@ -48,13 +49,19 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	}
 	sort.Sort(sortedKeys)
 
+	escape := func(s string) string {
+		buf := bytes.NewBuffer(make([]byte, 0, len(s)))
+		EscapeStringJSON(buf, s)
+		return buf.String()
+	}
+
 	nn, err := io.WriteString(w, "{")
 	if err != nil {
 		return n + int64(nn), err
 	}
 	n += int64(nn)
 	for i, k := range sortedKeys {
-		s := fmt.Sprintf("\"%s\":\"%s\",", k, c[k])
+		s := fmt.Sprintf("\"%s\":\"%s\",", escape(k), escape(c[k]))
 		if i == len(sortedKeys)-1 {
 			s = s[:len(s)-1] // remove last ','
 		}
@@ -73,6 +80,7 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 //
 // AppendTo sorts the context keys and writes the sorted
 // key-value pairs as canonical JSON object to w.
+// Sort order is based on the un-escaped keys.
 //
 // Note that neither keys nor values are escaped for JSON.
 func (c Context) AppendTo(dst []byte) (output []byte) {
@@ -87,9 +95,9 @@ func (c Context) AppendTo(dst []byte) (output []byte) {
 	if len(c) == 1 {
 		for k, v := range c {
 			out.WriteString(`{"`)
-			out.WriteString(k)
+			EscapeStringJSON(out, k)
 			out.WriteString(`":"`)
-			out.WriteString(v)
+			EscapeStringJSON(out, v)
 			out.WriteString(`"}`)
 		}
 		return out.Bytes()
@@ -104,9 +112,9 @@ func (c Context) AppendTo(dst []byte) (output []byte) {
 	out.WriteByte('{')
 	for i, k := range sortedKeys {
 		out.WriteByte('"')
-		out.WriteString(k)
+		EscapeStringJSON(out, k)
 		out.WriteString(`":"`)
-		out.WriteString(c[k])
+		EscapeStringJSON(out, c[k])
 		out.WriteByte('"')
 		if i < len(sortedKeys)-1 {
 			out.WriteByte(',')

cmd/crypto/kms_test.go

@@ -65,11 +65,15 @@ var contextWriteToTests = []struct {
 	Context      Context
 	ExpectedJSON string
 }{
-	{Context: Context{}, ExpectedJSON: "{}"},                                                    // 0
-	{Context: Context{"a": "b"}, ExpectedJSON: `{"a":"b"}`},                                     // 1
-	{Context: Context{"a": "b", "c": "d"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 2
-	{Context: Context{"c": "d", "a": "b"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 3
-	{Context: Context{"0": "1", "-": "2", ".": "#"}, ExpectedJSON: `{"-":"2",".":"#","0":"1"}`}, // 4
+	0: {Context: Context{}, ExpectedJSON: "{}"},
+	1: {Context: Context{"a": "b"}, ExpectedJSON: `{"a":"b"}`},
+	2: {Context: Context{"a": "b", "c": "d"}, ExpectedJSON: `{"a":"b","c":"d"}`},
+	3: {Context: Context{"c": "d", "a": "b"}, ExpectedJSON: `{"a":"b","c":"d"}`},
+	4: {Context: Context{"0": "1", "-": "2", ".": "#"}, ExpectedJSON: `{"-":"2",".":"#","0":"1"}`},
+	// rfc 8259 escapes
+	5: {Context: Context{"0": "1", "key\\": "val\tue\r\n", "\"": "\""}, ExpectedJSON: `{"\"":"\"","0":"1","key\\":"val\tue\r\n"}`},
+	// html sensitive escapes
+	6: {Context: Context{"a": "<>&"}, ExpectedJSON: `{"a":"\u003c\u003e\u0026"}`},
 }
 
 func TestContextWriteTo(t *testing.T) {
@@ -101,7 +105,7 @@ func TestContextAppendTo(t *testing.T) {
 }
 
 func BenchmarkContext_AppendTo(b *testing.B) {
-	tests := []Context{{}, {"bucket": "warp-benchmark-bucket"}, {"0": "1", "-": "2", ".": "#"}, {"34trg": "dfioutr89", "ikjfdghkjf": "jkedfhgfjkhg", "sdfhsdjkh": "if88889", "asddsirfh804": "kjfdshgdfuhgfg78-45604586#$%"}}
+	tests := []Context{{}, {"bucket": "warp-benchmark-bucket"}, {"0": "1", "-": "2", ".": "#"}, {"34trg": "dfioutr89", "ikjfdghkjf": "jkedfhgfjkhg", "sdfhsdjkh": "if88889", "asddsirfh804": "kjfdshgdfuhgfg78-45604586#$%<>&"}}
 	for _, test := range tests {
 		b.Run(fmt.Sprintf("%d-elems", len(test)), func(b *testing.B) {
 			dst := make([]byte, 0, 1024)

cmd/crypto/metadata.go

@@ -44,6 +44,15 @@ const (
 	// MetaDataEncryptionKey is the sealed data encryption key (DEK) received from
 	// the KMS.
 	MetaDataEncryptionKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
+
+	// MetaContext is the KMS context provided by a client when encrypting an
+	// object with SSE-KMS. A client may not send a context in which case the
+	// MetaContext will not be present.
+	// MetaContext only contains the bucket/object name if the client explicitly
+	// added it. However, when decrypting an object the bucket/object name must
+	// be part of the object. Therefore, the bucket/object name must be added
+	// to the context, if not present, whenever a decryption is performed.
+	MetaContext = "X-Minio-Internal-Server-Side-Encryption-Context"
 )
 
 // IsMultiPart returns true if the object metadata indicates
@@ -109,26 +118,35 @@ func IsSourceEncrypted(metadata map[string]string) bool {
 //
 // IsEncrypted only checks whether the metadata contains at least
 // one entry indicating SSE-C or SSE-S3.
-func IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[MetaIV]; ok {
-		return true
-	}
-	if _, ok := metadata[MetaAlgorithm]; ok {
-		return true
-	}
-	if IsMultiPart(metadata) {
-		return true
+func IsEncrypted(metadata map[string]string) (Type, bool) {
+	if S3KMS.IsEncrypted(metadata) {
+		return S3KMS, true
 	}
 	if S3.IsEncrypted(metadata) {
-		return true
+		return S3, true
 	}
 	if SSEC.IsEncrypted(metadata) {
-		return true
+		return SSEC, true
 	}
-	if S3KMS.IsEncrypted(metadata) {
-		return true
+	if IsMultiPart(metadata) {
+		return nil, true
 	}
-	return false
+	if _, ok := metadata[MetaIV]; ok {
+		return nil, true
+	}
+	if _, ok := metadata[MetaAlgorithm]; ok {
+		return nil, true
+	}
+	if _, ok := metadata[MetaKeyID]; ok {
+		return nil, true
+	}
+	if _, ok := metadata[MetaDataEncryptionKey]; ok {
+		return nil, true
+	}
+	if _, ok := metadata[MetaContext]; ok {
+		return nil, true
+	}
+	return nil, false
 }
 
 // CreateMultipartMetadata adds the multipart flag entry to metadata

cmd/crypto/metadata_test.go

@@ -59,7 +59,7 @@ var isEncryptedTests = []struct {
 
 func TestIsEncrypted(t *testing.T) {
 	for i, test := range isEncryptedTests {
-		if isEncrypted := IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+		if _, isEncrypted := IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
 			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
 		}
 	}
@@ -74,8 +74,8 @@ var s3IsEncryptedTests = []struct {
 	{Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}},                             // 2
 	{Encrypted: false, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                         // 3
 	{Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}},                            // 4
-	{Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}},                                  // 5
-	{Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                      // 6
+	{Encrypted: false, Metadata: map[string]string{MetaKeyID: ""}},                                 // 5
+	{Encrypted: false, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                     // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }

cmd/crypto/sse-kms.go

@@ -82,12 +82,6 @@ func (ssekms) IsEncrypted(metadata map[string]string) bool {
 	if _, ok := metadata[MetaSealedKeyKMS]; ok {
 		return true
 	}
-	if _, ok := metadata[MetaKeyID]; ok {
-		return true
-	}
-	if _, ok := metadata[MetaDataEncryptionKey]; ok {
-		return true
-	}
 	return false
 }
 
@@ -95,11 +89,14 @@ func (ssekms) IsEncrypted(metadata map[string]string) bool {
 // from the metadata using KMS and returns the decrypted object
 // key.
 func (s3 ssekms) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
-	keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata)
+	keyID, kmsKey, sealedKey, ctx, err := s3.ParseMetadata(metadata)
 	if err != nil {
 		return key, err
 	}
-	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if _, ok := ctx[bucket]; !ok {
+		ctx[bucket] = path.Join(bucket, object)
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, ctx)
 	if err != nil {
 		return key, err
 	}
@@ -147,19 +144,19 @@ func (ssekms) CreateMetadata(metadata map[string]string, keyID string, kmsKey []
 // KMS data key it returns both. If the metadata does not contain neither a
 // KMS master key ID nor a sealed KMS data key it returns an empty keyID and
 // KMS data key. Otherwise, it returns an error.
-func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, ctx Context, err error) {
 	// Extract all required values from object metadata
 	b64IV, ok := metadata[MetaIV]
 	if !ok {
-		return keyID, kmsKey, sealedKey, errMissingInternalIV
+		return keyID, kmsKey, sealedKey, ctx, errMissingInternalIV
 	}
 	algorithm, ok := metadata[MetaAlgorithm]
 	if !ok {
-		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+		return keyID, kmsKey, sealedKey, ctx, errMissingInternalSealAlgorithm
 	}
 	b64SealedKey, ok := metadata[MetaSealedKeyKMS]
 	if !ok {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
+		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
 	}
 
 	// There are two possibilites:
@@ -169,33 +166,44 @@ func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []
 	keyID, idPresent := metadata[MetaKeyID]
 	b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
 	if !idPresent && kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
+		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
 	}
 	if idPresent && !kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
 	}
 
 	// Check whether all extracted values are well-formed
 	iv, err := base64.StdEncoding.DecodeString(b64IV)
 	if err != nil || len(iv) != 32 {
-		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+		return keyID, kmsKey, sealedKey, ctx, errInvalidInternalIV
 	}
 	if algorithm != SealAlgorithm {
-		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+		return keyID, kmsKey, sealedKey, ctx, errInvalidInternalSealAlgorithm
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
+		return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed key for SSE-KMS is invalid")
 	}
 	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
 		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
 		if err != nil {
-			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
+			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS data key for SSE-KMS is invalid")
+		}
+	}
+	b64Ctx, ok := metadata[MetaContext]
+	if ok {
+		b, err := base64.StdEncoding.DecodeString(b64Ctx)
+		if err != nil {
+			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal KMS context is not base64-encoded")
+		}
+		var json = jsoniter.ConfigCompatibleWithStandardLibrary
+		if err = json.Unmarshal(b, ctx); err != nil {
+			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS context is invalid")
 		}
 	}
 
 	sealedKey.Algorithm = algorithm
 	copy(sealedKey.IV[:], iv)
 	copy(sealedKey.Key[:], encryptedKey)
-	return keyID, kmsKey, sealedKey, nil
+	return keyID, kmsKey, sealedKey, ctx, nil
 }

cmd/crypto/sse-s3.go

@@ -62,12 +62,6 @@ func (sses3) IsEncrypted(metadata map[string]string) bool {
 	if _, ok := metadata[MetaSealedKeyS3]; ok {
 		return true
 	}
-	if _, ok := metadata[MetaKeyID]; ok {
-		return true
-	}
-	if _, ok := metadata[MetaDataEncryptionKey]; ok {
-		return true
-	}
 	return false
 }
 

cmd/data-scanner.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"math"
 	"math/rand"
+	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -42,9 +43,9 @@ import (
 )
 
 const (
-	dataCrawlSleepPerFolder  = time.Millisecond // Time to wait between folders.
-	dataCrawlStartDelay      = 1 * time.Minute  // Time to wait on startup and between cycles.
-	dataUsageUpdateDirCycles = 16               // Visit all folders every n cycles.
+	dataScannerSleepPerFolder = time.Millisecond // Time to wait between folders.
+	dataScannerStartDelay     = 1 * time.Minute  // Time to wait on startup and between cycles.
+	dataUsageUpdateDirCycles  = 16               // Visit all folders every n cycles.
 
 	healDeleteDangling    = true
 	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
@@ -55,27 +56,28 @@ var (
 	globalHealConfig   heal.Config
 	globalHealConfigMu sync.Mutex
 
-	dataCrawlerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+	dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 	// Sleeper values are updated when config is loaded.
-	crawlerSleeper = newDynamicSleeper(10, 10*time.Second)
+	scannerSleeper = newDynamicSleeper(10, 10*time.Second)
 )
 
-// initDataCrawler will start the crawler in the background.
-func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
-	go runDataCrawler(ctx, objAPI)
+// initDataScanner will start the scanner in the background.
+func initDataScanner(ctx context.Context, objAPI ObjectLayer) {
+	go runDataScanner(ctx, objAPI)
 }
 
-// runDataCrawler will start a data crawler.
+// runDataScanner will start a data scanner.
 // The function will block until the context is canceled.
-// There should only ever be one crawler running per cluster.
-func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
-	// Make sure only 1 crawler is running on the cluster.
-	locker := objAPI.NewNSLock(minioMetaBucket, "runDataCrawler.lock")
+// There should only ever be one scanner running per cluster.
+func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
+	var err error
+	// Make sure only 1 scanner is running on the cluster.
+	locker := objAPI.NewNSLock(minioMetaBucket, "runDataScanner.lock")
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	for {
-		err := locker.GetLock(ctx, dataCrawlerLeaderLockTimeout)
+		ctx, err = locker.GetLock(ctx, dataScannerLeaderLockTimeout)
 		if err != nil {
-			time.Sleep(time.Duration(r.Float64() * float64(dataCrawlStartDelay)))
+			time.Sleep(time.Duration(r.Float64() * float64(dataScannerStartDelay)))
 			continue
 		}
 		break
@@ -84,31 +86,34 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 
 	// Load current bloom cycle
 	nextBloomCycle := intDataUpdateTracker.current() + 1
-	var buf bytes.Buffer
-	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageBloomName, 0, -1, &buf, "", ObjectOptions{})
+
+	br, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageBloomName, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
 			logger.LogIf(ctx, err)
 		}
 	} else {
-		if buf.Len() == 8 {
-			nextBloomCycle = binary.LittleEndian.Uint64(buf.Bytes())
+		if br.ObjInfo.Size == 8 {
+			if err = binary.Read(br, binary.LittleEndian, &nextBloomCycle); err != nil {
+				logger.LogIf(ctx, err)
+			}
 		}
+		br.Close()
 	}
 
-	crawlTimer := time.NewTimer(dataCrawlStartDelay)
-	defer crawlTimer.Stop()
+	scannerTimer := time.NewTimer(dataScannerStartDelay)
+	defer scannerTimer.Stop()
 
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-crawlTimer.C:
+		case <-scannerTimer.C:
 			// Reset the timer for next cycle.
-			crawlTimer.Reset(dataCrawlStartDelay)
+			scannerTimer.Reset(dataScannerStartDelay)
 
 			if intDataUpdateTracker.debug {
-				console.Debugln("starting crawler cycle")
+				console.Debugln("starting scanner cycle")
 			}
 
 			// Wait before starting next cycle and wait on startup.
@@ -116,7 +121,7 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
 			logger.LogIf(ctx, err)
-			err = objAPI.CrawlAndGetDataUsage(ctx, bf, results)
+			err = objAPI.NSScanner(ctx, bf, results)
 			close(results)
 			logger.LogIf(ctx, err)
 			if err == nil {
@@ -124,13 +129,13 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 				nextBloomCycle++
 				var tmp [8]byte
 				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
-				r, err := hash.NewReader(bytes.NewReader(tmp[:]), int64(len(tmp)), "", "", int64(len(tmp)), false)
+				r, err := hash.NewReader(bytes.NewReader(tmp[:]), int64(len(tmp)), "", "", int64(len(tmp)))
 				if err != nil {
 					logger.LogIf(ctx, err)
 					continue
 				}
 
-				_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r, nil, nil), ObjectOptions{})
+				_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r), ObjectOptions{})
 				if !isErrBucketNotFound(err) {
 					logger.LogIf(ctx, err)
 				}
@@ -152,27 +157,27 @@ type folderScanner struct {
 	newCache   dataUsageCache
 	withFilter *bloomFilter
 
-	dataUsageCrawlDebug bool
-	healFolderInclude   uint32 // Include a clean folder one in n cycles.
-	healObjectSelect    uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
+	dataUsageScannerDebug bool
+	healFolderInclude     uint32 // Include a clean folder one in n cycles.
+	healObjectSelect      uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 
 	newFolders      []cachedFolder
 	existingFolders []cachedFolder
 	disks           []StorageAPI
 }
 
-// crawlDataFolder will crawl the basepath+cache.Info.Name and return an updated cache.
+// scanDataFolder will scanner the basepath+cache.Info.Name and return an updated cache.
 // The returned cache will always be valid, but may not be updated from the existing.
-// Before each operation sleepDuration is called which can be used to temporarily halt the crawler.
+// Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
-func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
+func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
 	t := UTCNow()
 
 	logPrefix := color.Green("data-usage: ")
 	logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
 	if intDataUpdateTracker.debug {
 		defer func() {
-			console.Debugf(logPrefix+" Crawl time: %v %s\n", time.Since(t), logSuffix)
+			console.Debugf(logPrefix+" Scanner time: %v %s\n", time.Since(t), logSuffix)
 		}()
 
 	}
@@ -182,16 +187,18 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		return cache, errors.New("internal error: root scan attempted")
 	}
 
+	skipHeal := cache.Info.SkipHealing
+
 	s := folderScanner{
-		root:                basePath,
-		getSize:             getSize,
-		oldCache:            cache,
-		newCache:            dataUsageCache{Info: cache.Info},
-		newFolders:          nil,
-		existingFolders:     nil,
-		dataUsageCrawlDebug: intDataUpdateTracker.debug,
-		healFolderInclude:   0,
-		healObjectSelect:    0,
+		root:                  basePath,
+		getSize:               getSize,
+		oldCache:              cache,
+		newCache:              dataUsageCache{Info: cache.Info},
+		newFolders:            nil,
+		existingFolders:       nil,
+		dataUsageScannerDebug: intDataUpdateTracker.debug,
+		healFolderInclude:     0,
+		healObjectSelect:      0,
 	}
 
 	// Add disks for set healing.
@@ -221,21 +228,21 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 			s.withFilter = nil
 		}
 	}
-	if s.dataUsageCrawlDebug {
-		console.Debugf(logPrefix+"Start crawling. Bloom filter: %v %s\n", s.withFilter != nil, logSuffix)
+	if s.dataUsageScannerDebug {
+		console.Debugf(logPrefix+"Start scanning. Bloom filter: %v %s\n", s.withFilter != nil, logSuffix)
 	}
 
 	done := ctx.Done()
 	var flattenLevels = 2
 
-	if s.dataUsageCrawlDebug {
+	if s.dataUsageScannerDebug {
 		console.Debugf(logPrefix+"Cycle: %v, Entries: %v %s\n", cache.Info.NextCycle, len(cache.Cache), logSuffix)
 	}
 
 	// Always scan flattenLevels deep. Cache root is level 0.
 	todo := []cachedFolder{{name: cache.Info.Name, objectHealProbDiv: 1}}
 	for i := 0; i < flattenLevels; i++ {
-		if s.dataUsageCrawlDebug {
+		if s.dataUsageScannerDebug {
 			console.Debugf(logPrefix+"Level %v, scanning %v directories. %s\n", i, len(todo), logSuffix)
 		}
 		select {
@@ -244,14 +251,14 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		default:
 		}
 		var err error
-		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1)
+		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1, skipHeal)
 		if err != nil {
 			// No useful information...
 			return cache, err
 		}
 	}
 
-	if s.dataUsageCrawlDebug {
+	if s.dataUsageScannerDebug {
 		console.Debugf(logPrefix+"New folders: %v %s\n", s.newFolders, logSuffix)
 	}
 
@@ -262,7 +269,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 			return s.newCache, ctx.Err()
 		default:
 		}
-		du, err := s.deepScanFolder(ctx, folder)
+		du, err := s.deepScanFolder(ctx, folder, skipHeal)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			continue
@@ -280,7 +287,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		}
 	}
 
-	if s.dataUsageCrawlDebug {
+	if s.dataUsageScannerDebug {
 		console.Debugf(logPrefix+"Existing folders: %v %s\n", len(s.existingFolders), logSuffix)
 	}
 
@@ -307,13 +314,13 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 				// If folder isn't in filter, skip it completely.
 				if !s.withFilter.containsDir(folder.name) {
 					if !h.mod(s.oldCache.Info.NextCycle, s.healFolderInclude/folder.objectHealProbDiv) {
-						if s.dataUsageCrawlDebug {
+						if s.dataUsageScannerDebug {
 							console.Debugf(logPrefix+"Skipping non-updated folder: %v %s\n", folder, logSuffix)
 						}
 						s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
 						continue
 					} else {
-						if s.dataUsageCrawlDebug {
+						if s.dataUsageScannerDebug {
 							console.Debugf(logPrefix+"Adding non-updated folder to heal check: %v %s\n", folder.name, logSuffix)
 						}
 						// Update probability of including objects
@@ -324,7 +331,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		}
 
 		// Update on this cycle...
-		du, err := s.deepScanFolder(ctx, folder)
+		du, err := s.deepScanFolder(ctx, folder, skipHeal)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			continue
@@ -335,8 +342,8 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		}
 		s.newCache.replaceHashed(h, folder.parent, *du)
 	}
-	if s.dataUsageCrawlDebug {
-		console.Debugf(logPrefix+"Finished crawl, %v entries %s\n", len(s.newCache.Cache), logSuffix)
+	if s.dataUsageScannerDebug {
+		console.Debugf(logPrefix+"Finished scanner, %v entries %s\n", len(s.newCache.Cache), logSuffix)
 	}
 	s.newCache.Info.LastUpdate = UTCNow()
 	s.newCache.Info.NextCycle++
@@ -347,7 +354,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 // Files found in the folders will be added to f.newCache.
 // If final is provided folders will be put into f.newFolders or f.existingFolders.
 // If final is not provided the folders found are returned from the function.
-func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
+func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool, skipHeal bool) ([]cachedFolder, error) {
 	var nextFolders []cachedFolder
 	done := ctx.Done()
 	scannerLogPrefix := color.Green("folder-scanner:")
@@ -365,7 +372,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 		var activeLifeCycle *lifecycle.Lifecycle
 		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
-			if f.dataUsageCrawlDebug {
+			if f.dataUsageScannerDebug {
 				console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
 			}
 			activeLifeCycle = f.oldCache.Info.lifeCycle
@@ -376,20 +383,20 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
 				if !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
 					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
-					if f.dataUsageCrawlDebug {
+					if f.dataUsageScannerDebug {
 						console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
 					}
 					continue
 				} else {
-					if f.dataUsageCrawlDebug {
+					if f.dataUsageScannerDebug {
 						console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
 					}
-					// If probability was already crawlerHealFolderInclude, keep it.
+					// If probability was already scannerHealFolderInclude, keep it.
 					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
 		}
-		crawlerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
+		scannerSleeper.Sleep(ctx, dataScannerSleepPerFolder)
 
 		cache := dataUsageEntry{}
 
@@ -398,22 +405,22 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			entName = path.Clean(path.Join(folder.name, entName))
 			bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
 			if bucket == "" {
-				if f.dataUsageCrawlDebug {
+				if f.dataUsageScannerDebug {
 					console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName)
 				}
-				return nil
+				return errDoneForNow
 			}
 
 			if isReservedOrInvalidBucket(bucket, false) {
-				if f.dataUsageCrawlDebug {
+				if f.dataUsageScannerDebug {
 					console.Debugf(scannerLogPrefix+" invalid bucket: %v, entry: %v\n", bucket, entName)
 				}
-				return nil
+				return errDoneForNow
 			}
 
 			select {
 			case <-done:
-				return ctx.Err()
+				return errDoneForNow
 			default:
 			}
 
@@ -441,20 +448,25 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			}
 
 			// Dynamic time delay.
-			wait := crawlerSleeper.Timer(ctx)
+			wait := scannerSleeper.Timer(ctx)
 
 			// Get file size, ignore errors.
-			item := crawlItem{
+			item := scannerItem{
 				Path:       path.Join(f.root, entName),
 				Typ:        typ,
 				bucket:     bucket,
 				prefix:     path.Dir(prefix),
 				objectName: path.Base(entName),
-				debug:      f.dataUsageCrawlDebug,
+				debug:      f.dataUsageScannerDebug,
 				lifeCycle:  activeLifeCycle,
 				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
 
+			// if the drive belongs to an erasure set
+			// that is already being healed, skip the
+			// healing attempt on this drive.
+			item.heal = item.heal && !skipHeal
+
 			sizeSummary, err := f.getSize(item)
 			if err == errSkipFile {
 				wait() // wait to proceed to next entry.
@@ -521,17 +533,18 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		healObjectsPrefix := color.Green("healObjects:")
 		for k := range existing {
 			bucket, prefix := path2BucketObject(k)
-			if f.dataUsageCrawlDebug {
+			if f.dataUsageScannerDebug {
 				console.Debugf(scannerLogPrefix+" checking disappeared folder: %v/%v\n", bucket, prefix)
 			}
 
 			// Dynamic time delay.
-			wait := crawlerSleeper.Timer(ctx)
+			wait := scannerSleeper.Timer(ctx)
 			resolver.bucket = bucket
 
 			foundObjs := false
 			dangling := false
 			ctx, cancel := context.WithCancel(ctx)
+
 			err := listPathRaw(ctx, listPathRawOptions{
 				disks:          f.disks,
 				bucket:         bucket,
@@ -541,13 +554,13 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				minDisks:       len(f.disks), // We want full consistency.
 				// Weird, maybe transient error.
 				agreed: func(entry metaCacheEntry) {
-					if f.dataUsageCrawlDebug {
+					if f.dataUsageScannerDebug {
 						console.Debugf(healObjectsPrefix+" got agreement: %v\n", entry.name)
 					}
 				},
 				// Some disks have data for this.
 				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
-					if f.dataUsageCrawlDebug {
+					if f.dataUsageScannerDebug {
 						console.Debugf(healObjectsPrefix+" got partial, %d agreed, errs: %v\n", nAgreed, errs)
 					}
 
@@ -556,7 +569,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 
 					// Sleep and reset.
 					wait()
-					wait = crawlerSleeper.Timer(ctx)
+					wait = scannerSleeper.Timer(ctx)
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
 						for _, err := range errs {
@@ -569,7 +582,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 						entry, _ = entries.firstFound()
 					}
 
-					if f.dataUsageCrawlDebug {
+					if f.dataUsageScannerDebug {
 						console.Debugf(healObjectsPrefix+" resolved to: %v, dir: %v\n", entry.name, entry.isDir())
 					}
 
@@ -593,7 +606,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 					for _, ver := range fiv.Versions {
 						// Sleep and reset.
 						wait()
-						wait = crawlerSleeper.Timer(ctx)
+						wait = scannerSleeper.Timer(ctx)
 						err := bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
 							object:    fiv.Name,
@@ -607,31 +620,31 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				},
 				// Too many disks failed.
 				finished: func(errs []error) {
-					if f.dataUsageCrawlDebug {
+					if f.dataUsageScannerDebug {
 						console.Debugf(healObjectsPrefix+" too many errors: %v\n", errs)
 					}
 					cancel()
 				},
 			})
 
-			if f.dataUsageCrawlDebug && err != nil && err != errFileNotFound {
+			if f.dataUsageScannerDebug && err != nil && err != errFileNotFound {
 				console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err)
 			}
 
 			// If we found one or more disks with this folder, delete it.
 			if err == nil && dangling {
-				if f.dataUsageCrawlDebug {
+				if f.dataUsageScannerDebug {
 					console.Debugf(healObjectsPrefix+" deleting dangling directory %s\n", prefix)
 				}
 
 				objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{
 					Recursive: true,
-					Remove:    true,
+					Remove:    healDeleteDangling,
 				},
 					func(bucket, object, versionID string) error {
-						// Wait for each heal as per crawler frequency.
+						// Wait for each heal as per scanner frequency.
 						wait()
-						wait = crawlerSleeper.Timer(ctx)
+						wait = scannerSleeper.Timer(ctx)
 						return bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
 							object:    object,
@@ -659,7 +672,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 }
 
 // deepScanFolder will deep scan a folder and return the size if no error occurs.
-func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder) (*dataUsageEntry, error) {
+func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder, skipHeal bool) (*dataUsageEntry, error) {
 	var cache dataUsageEntry
 
 	done := ctx.Done()
@@ -671,7 +684,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 	addDir = func(entName string, typ os.FileMode) error {
 		select {
 		case <-done:
-			return ctx.Err()
+			return errDoneForNow
 		default:
 		}
 
@@ -679,12 +692,12 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 			dirStack = append(dirStack, entName)
 			err := readDirFn(path.Join(dirStack...), addDir)
 			dirStack = dirStack[:len(dirStack)-1]
-			crawlerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
+			scannerSleeper.Sleep(ctx, dataScannerSleepPerFolder)
 			return err
 		}
 
 		// Dynamic time delay.
-		wait := crawlerSleeper.Timer(ctx)
+		wait := scannerSleeper.Timer(ctx)
 
 		// Get file size, ignore errors.
 		dirStack = append(dirStack, entName)
@@ -694,24 +707,29 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 		bucket, prefix := path2BucketObjectWithBasePath(f.root, fileName)
 		var activeLifeCycle *lifecycle.Lifecycle
 		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
-			if f.dataUsageCrawlDebug {
+			if f.dataUsageScannerDebug {
 				console.Debugf(deepScannerLogPrefix+" Prefix %q has active rules\n", prefix)
 			}
 			activeLifeCycle = f.oldCache.Info.lifeCycle
 		}
 
-		sizeSummary, err := f.getSize(
-			crawlItem{
-				Path:       fileName,
-				Typ:        typ,
-				bucket:     bucket,
-				prefix:     path.Dir(prefix),
-				objectName: path.Base(entName),
-				debug:      f.dataUsageCrawlDebug,
-				lifeCycle:  activeLifeCycle,
-				heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
-			})
+		item := scannerItem{
+			Path:       fileName,
+			Typ:        typ,
+			bucket:     bucket,
+			prefix:     path.Dir(prefix),
+			objectName: path.Base(entName),
+			debug:      f.dataUsageScannerDebug,
+			lifeCycle:  activeLifeCycle,
+			heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
+		}
 
+		// if the drive belongs to an erasure set
+		// that is already being healed, skip the
+		// healing attempt on this drive.
+		item.heal = item.heal && !skipHeal
+
+		sizeSummary, err := f.getSize(item)
 		if err == errSkipFile {
 			// Wait to throttle IO
 			wait()
@@ -736,8 +754,8 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 	return &cache, nil
 }
 
-// crawlItem represents each file while walking.
-type crawlItem struct {
+// scannerItem represents each file while walking.
+type scannerItem struct {
 	Path string
 	Typ  os.FileMode
 
@@ -757,10 +775,10 @@ type sizeSummary struct {
 	replicaSize    int64
 }
 
-type getSizeFn func(item crawlItem) (sizeSummary, error)
+type getSizeFn func(item scannerItem) (sizeSummary, error)
 
 // transformMetaDir will transform a directory to prefix/file.ext
-func (i *crawlItem) transformMetaDir() {
+func (i *scannerItem) transformMetaDir() {
 	split := strings.Split(i.prefix, SlashSeparator)
 	if len(split) > 1 {
 		i.prefix = path.Join(split[:len(split)-1]...)
@@ -773,22 +791,21 @@ func (i *crawlItem) transformMetaDir() {
 
 // actionMeta contains information used to apply actions.
 type actionMeta struct {
-	oi               ObjectInfo
-	successorModTime time.Time // The modtime of the successor version
-	numVersions      int       // The number of versions of this object
-	bitRotScan       bool      // indicates if bitrot check was requested.
+	oi         ObjectInfo
+	bitRotScan bool // indicates if bitrot check was requested.
 }
 
+var applyActionsLogPrefix = color.Green("applyActions:")
+
 // applyActions will apply lifecycle checks on to a scanned item.
 // The resulting size on disk will always be returned.
 // The metadata will be compared to consensus on the object layer before any changes are applied.
 // If no metadata is supplied, -1 is returned if no action is taken.
-func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta actionMeta) (size int64) {
+func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, meta actionMeta) (size int64) {
 	size, err := meta.oi.GetActualSize()
 	if i.debug {
 		logger.LogIf(ctx, err)
 	}
-	applyActionsLogPrefix := color.Green("applyActions:")
 	if i.heal {
 		if i.debug {
 			if meta.oi.VersionID != "" {
@@ -827,8 +844,8 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			VersionID:        meta.oi.VersionID,
 			DeleteMarker:     meta.oi.DeleteMarker,
 			IsLatest:         meta.oi.IsLatest,
-			NumVersions:      meta.numVersions,
-			SuccessorModTime: meta.successorModTime,
+			NumVersions:      meta.oi.NumVersions,
+			SuccessorModTime: meta.oi.SuccessorModTime,
 			RestoreOngoing:   meta.oi.RestoreOngoing,
 			RestoreExpires:   meta.oi.RestoreExpires,
 			TransitionStatus: meta.oi.TransitionStatus,
@@ -863,8 +880,8 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 				logger.LogIf(ctx, err)
 				return size
 			}
-		case ObjectNotFound:
-			// object not found return 0
+		case ObjectNotFound, VersionNotFound:
+			// object not found or version not found return 0
 			return 0
 		default:
 			// All other errors proceed.
@@ -872,87 +889,135 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			return size
 		}
 	}
-	size = obj.Size
 
-	// Recalculate action.
+	var applied bool
+	action = evalActionFromLifecycle(ctx, *i.lifeCycle, obj, i.debug)
+	if action != lifecycle.NoneAction {
+		applied = applyLifecycleAction(ctx, action, o, obj)
+	}
+
+	if applied {
+		switch action {
+		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+		default: // for all lifecycle actions that remove data
+			return 0
+		}
+	}
+	return size
+}
+
+func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, obj ObjectInfo, debug bool) (action lifecycle.Action) {
 	lcOpts := lifecycle.ObjectOpts{
-		Name:             i.objectPath(),
+		Name:             obj.Name,
 		UserTags:         obj.UserTags,
 		ModTime:          obj.ModTime,
 		VersionID:        obj.VersionID,
 		DeleteMarker:     obj.DeleteMarker,
 		IsLatest:         obj.IsLatest,
-		NumVersions:      meta.numVersions,
-		SuccessorModTime: meta.successorModTime,
+		NumVersions:      obj.NumVersions,
+		SuccessorModTime: obj.SuccessorModTime,
 		RestoreOngoing:   obj.RestoreOngoing,
 		RestoreExpires:   obj.RestoreExpires,
 		TransitionStatus: obj.TransitionStatus,
 	}
-	action = i.lifeCycle.ComputeAction(lcOpts)
-	if i.debug {
+
+	action = lc.ComputeAction(lcOpts)
+	if debug {
 		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
 	}
-	switch action {
-	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
-	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-	default:
-		// No action.
-		return size
+
+	if action == lifecycle.NoneAction {
+		return action
 	}
 
-	opts := ObjectOptions{}
 	switch action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
 		// Defensive code, should never happen
 		if obj.VersionID == "" {
-			return size
+			return lifecycle.NoneAction
 		}
-		if rcfg, _ := globalBucketObjectLockSys.Get(i.bucket); rcfg.LockEnabled {
+		if rcfg, _ := globalBucketObjectLockSys.Get(obj.Bucket); rcfg.LockEnabled {
 			locked := enforceRetentionForDeletion(ctx, obj)
 			if locked {
-				if i.debug {
+				if debug {
 					if obj.VersionID != "" {
-						console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", i.objectPath(), obj.VersionID)
+						console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID)
 					} else {
-						console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", i.objectPath())
+						console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
 					}
 				}
-				return size
+				return lifecycle.NoneAction
 			}
 		}
+	}
+
+	return action
+}
+
+func applyTransitionAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) bool {
+	opts := ObjectOptions{}
+	if obj.TransitionStatus == "" {
+		opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
 		opts.VersionID = obj.VersionID
-	case lifecycle.DeleteAction, lifecycle.DeleteRestoredAction:
-		opts.Versioned = globalBucketVersioningSys.Enabled(i.bucket)
-	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		if obj.TransitionStatus == "" {
-			opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
-			opts.VersionID = obj.VersionID
-			opts.TransitionStatus = lifecycle.TransitionPending
-			if _, err = o.DeleteObject(ctx, obj.Bucket, obj.Name, opts); err != nil {
-				// Assume it is still there.
-				logger.LogIf(ctx, err)
-				return size
+		opts.TransitionStatus = lifecycle.TransitionPending
+		if _, err := objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts); err != nil {
+			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+				return false
 			}
-		}
-		globalTransitionState.queueTransitionTask(obj)
-		return 0
-	}
-
-	if obj.TransitionStatus != "" {
-		if err := deleteTransitionedObject(ctx, o, i.bucket, i.objectPath(), lcOpts, action, false); err != nil {
+			// Assume it is still there.
 			logger.LogIf(ctx, err)
-			return size
+			return false
 		}
-		// Notification already sent at *deleteTransitionedObject*, return '0' here.
-		return 0
+	}
+	globalTransitionState.queueTransitionTask(obj)
+	return true
+
+}
+
+func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject bool) bool {
+	lcOpts := lifecycle.ObjectOpts{
+		Name:             obj.Name,
+		UserTags:         obj.UserTags,
+		ModTime:          obj.ModTime,
+		VersionID:        obj.VersionID,
+		DeleteMarker:     obj.DeleteMarker,
+		IsLatest:         obj.IsLatest,
+		NumVersions:      obj.NumVersions,
+		SuccessorModTime: obj.SuccessorModTime,
+		RestoreOngoing:   obj.RestoreOngoing,
+		RestoreExpires:   obj.RestoreExpires,
+		TransitionStatus: obj.TransitionStatus,
 	}
 
-	obj, err = o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
+	if err := deleteTransitionedObject(ctx, objLayer, obj.Bucket, obj.Name, lcOpts, restoredObject, false); err != nil {
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			return false
+		}
+		logger.LogIf(ctx, err)
+		return false
+	}
+	// Notification already sent at *deleteTransitionedObject*, just return 'true' here.
+	return true
+}
+
+func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool {
+	opts := ObjectOptions{}
+
+	if applyOnVersion {
+		opts.VersionID = obj.VersionID
+	}
+	if opts.VersionID == "" {
+		opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
+	}
+
+	obj, err := objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts)
 	if err != nil {
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			return false
+		}
 		// Assume it is still there.
 		logger.LogIf(ctx, err)
-		return size
+		return false
 	}
 
 	eventName := event.ObjectRemovedDelete
@@ -963,20 +1028,42 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	// Notify object deleted event.
 	sendEvent(eventArgs{
 		EventName:  eventName,
-		BucketName: i.bucket,
+		BucketName: obj.Bucket,
 		Object:     obj,
 		Host:       "Internal: [ILM-EXPIRY]",
 	})
-	return 0
+
+	return true
+}
+
+// Apply object, object version, restored object or restored object version action on the given object
+func applyExpiryRule(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
+	if obj.TransitionStatus != "" {
+		return applyExpiryOnTransitionedObject(ctx, objLayer, obj, restoredObject)
+	}
+	return applyExpiryOnNonTransitionedObjects(ctx, objLayer, obj, applyOnVersion)
+}
+
+// Perform actions (removal or transitioning of objects), return true the action is successfully performed
+func applyLifecycleAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) (success bool) {
+	switch action {
+	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
+		success = applyExpiryRule(ctx, objLayer, obj, false, action == lifecycle.DeleteVersionAction)
+	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		success = applyExpiryRule(ctx, objLayer, obj, true, action == lifecycle.DeleteRestoredVersionAction)
+	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+		success = applyTransitionAction(ctx, action, objLayer, obj)
+	}
+	return
 }
 
 // objectPath returns the prefix and object name.
-func (i *crawlItem) objectPath() string {
+func (i *scannerItem) objectPath() string {
 	return path.Join(i.prefix, i.objectName)
 }
 
 // healReplication will heal a scanned item that has failed replication.
-func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
+func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		// heal delete marker replication failure or versioned delete replication failure
 		if oi.ReplicationStatus == replication.Pending ||
@@ -989,10 +1076,10 @@ func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, oi Objec
 	switch oi.ReplicationStatus {
 	case replication.Pending:
 		sizeS.pendingSize += oi.Size
-		globalReplicationState.queueReplicaTask(oi)
+		globalReplicationPool.queueReplicaTask(oi)
 	case replication.Failed:
 		sizeS.failedSize += oi.Size
-		globalReplicationState.queueReplicaTask(oi)
+		globalReplicationPool.queueReplicaTask(oi)
 	case replication.Completed, "COMPLETE":
 		sizeS.replicatedSize += oi.Size
 	case replication.Replica:
@@ -1001,7 +1088,7 @@ func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, oi Objec
 }
 
 // healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
-func (i *crawlItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
+func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
 	// handle soft delete and permanent delete failures here.
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		versionID := ""
@@ -1011,7 +1098,7 @@ func (i *crawlItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, o
 		} else {
 			versionID = oi.VersionID
 		}
-		globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
+		globalReplicationPool.queueReplicaDeleteTask(DeletedObjectVersionInfo{
 			DeletedObject: DeletedObject{
 				ObjectName:                    oi.Name,
 				DeleteMarkerVersionID:         dmVersionID,

cmd/data-update-tracker.go

@@ -80,7 +80,7 @@ func newDataUpdateTracker() *dataUpdateTracker {
 		Current: dataUpdateFilter{
 			idx: 1,
 		},
-		debug:      env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn || serverDebugLog,
+		debug:      env.Get(envDataUsageScannerDebug, config.EnableOff) == config.EnableOn || serverDebugLog,
 		input:      make(chan string, dataUpdateTrackerQueueSize),
 		save:       make(chan struct{}, 1),
 		saveExited: make(chan struct{}),

cmd/data-usage-cache.go

@@ -88,9 +88,12 @@ type dataUsageEntryInfo struct {
 
 type dataUsageCacheInfo struct {
 	// Name of the bucket. Also root element.
-	Name        string
-	LastUpdate  time.Time
-	NextCycle   uint32
+	Name       string
+	LastUpdate time.Time
+	NextCycle  uint32
+	// indicates if the disk is being healed and scanner
+	// should skip healing the disk
+	SkipHealing bool
 	BloomFilter []byte               `msg:"BloomFilter,omitempty"`
 	lifeCycle   *lifecycle.Lifecycle `msg:"-"`
 }
@@ -482,7 +485,7 @@ type objectIO interface {
 // Only backend errors are returned as errors.
 // If the object is not found or unable to deserialize d is cleared and nil error is returned.
 func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
-	r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, readLock, ObjectOptions{})
+	r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, noLock, ObjectOptions{})
 	if err != nil {
 		switch err.(type) {
 		case ObjectNotFound:
@@ -510,7 +513,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 	}()
 	defer pr.Close()
 
-	r, err := hash.NewReader(pr, -1, "", "", -1, false)
+	r, err := hash.NewReader(pr, -1, "", "", -1)
 	if err != nil {
 		return err
 	}
@@ -518,8 +521,8 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 	_, err = store.PutObject(ctx,
 		dataUsageBucket,
 		name,
-		NewPutObjReader(r, nil, nil),
-		ObjectOptions{})
+		NewPutObjReader(r),
+		ObjectOptions{NoLock: true})
 	if isErrBucketNotFound(err) {
 		return nil
 	}

cmd/data-usage-cache_gen.go

@@ -313,6 +313,12 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "NextCycle")
 				return
 			}
+		case "SkipHealing":
+			z.SkipHealing, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "SkipHealing")
+				return
+			}
 		case "BloomFilter":
 			z.BloomFilter, err = dc.ReadBytes(z.BloomFilter)
 			if err != nil {
@@ -333,11 +339,11 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	// omitempty: check for empty values
-	zb0001Len := uint32(4)
-	var zb0001Mask uint8 /* 4 bits */
+	zb0001Len := uint32(5)
+	var zb0001Mask uint8 /* 5 bits */
 	if z.BloomFilter == nil {
 		zb0001Len--
-		zb0001Mask |= 0x8
+		zb0001Mask |= 0x10
 	}
 	// variable map header, size zb0001Len
 	err = en.Append(0x80 | uint8(zb0001Len))
@@ -377,7 +383,17 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "NextCycle")
 		return
 	}
-	if (zb0001Mask & 0x8) == 0 { // if not empty
+	// write "SkipHealing"
+	err = en.Append(0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.SkipHealing)
+	if err != nil {
+		err = msgp.WrapError(err, "SkipHealing")
+		return
+	}
+	if (zb0001Mask & 0x10) == 0 { // if not empty
 		// write "BloomFilter"
 		err = en.Append(0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
 		if err != nil {
@@ -396,11 +412,11 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// omitempty: check for empty values
-	zb0001Len := uint32(4)
-	var zb0001Mask uint8 /* 4 bits */
+	zb0001Len := uint32(5)
+	var zb0001Mask uint8 /* 5 bits */
 	if z.BloomFilter == nil {
 		zb0001Len--
-		zb0001Mask |= 0x8
+		zb0001Mask |= 0x10
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
@@ -416,7 +432,10 @@ func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "NextCycle"
 	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
 	o = msgp.AppendUint32(o, z.NextCycle)
-	if (zb0001Mask & 0x8) == 0 { // if not empty
+	// string "SkipHealing"
+	o = append(o, 0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67)
+	o = msgp.AppendBool(o, z.SkipHealing)
+	if (zb0001Mask & 0x10) == 0 { // if not empty
 		// string "BloomFilter"
 		o = append(o, 0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
 		o = msgp.AppendBytes(o, z.BloomFilter)
@@ -460,6 +479,12 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "NextCycle")
 				return
 			}
+		case "SkipHealing":
+			z.SkipHealing, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "SkipHealing")
+				return
+			}
 		case "BloomFilter":
 			z.BloomFilter, bts, err = msgp.ReadBytesBytes(bts, z.BloomFilter)
 			if err != nil {
@@ -480,7 +505,7 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCacheInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint32Size + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint32Size + 12 + msgp.BoolSize + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
 	return
 }