avoid unnecessary KMS requests during single-part PUT (#9220)

This commit fixes a performance issue caused
by too many calls to the external KMS - i.e.
for single-part PUT requests.

In general, the issue is caused by a sub-optimal
code structure. In particular, when the server
encrypts an object it requests a new data encryption
key from the KMS. With this key it does some key
derivation and encrypts the object content and
ETag.

However, to behave S3-compatible the MinIO server
has to return the plaintext ETag to the client
in case SSE-S3.
Therefore, the server code used to decrypt the
(previously encrypted) ETag again by requesting
the data encryption key (KMS decrypt API) from
the KMS.

This leads to 2 KMS API calls (1 generate key and
1 decrypt key) per PUT operation - while only
one KMS call is necessary.

This commit fixes this by fetching a data key only
once from the KMS and keeping the derived object
encryption key around (for the lifetime of the request).

This leads to a significant performance improvement
w.r.t. to PUT workloads:
```
Operation: PUT
Operations: 161 -> 239
Duration: 28s -> 29s
* Average: +47.56% (+25.8 MiB/s) throughput, +47.56% (+2.6) obj/s
* Fastest: +55.49% (+34.5 MiB/s) throughput, +55.49% (+3.5) obj/s
* 50% Median: +58.24% (+32.8 MiB/s) throughput, +58.24% (+3.3) obj/s
* Slowest: +1.83% (+0.6 MiB/s) throughput, +1.83% (+0.1) obj/s
```

.jshintrc

@@ -1,4 +0,0 @@
-{
-  "asi": true,
-  "esnext": true   
-}

.mention-bot

@@ -1,4 +0,0 @@
-{
-    "numFilesToCheck": 10,
-    "requiredOrgs": ["minio"]
-}

.travis.yml

@@ -32,8 +32,7 @@ matrix:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
         - diff -au <(gofmt -s -d pkg) <(printf "")
-        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done
-        - make verifiers
+        - make test-race
         - make crosscompile
         - make verify
         - cd browser && npm install && npm run test && cd ..

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# MinIO Contribution Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
+# MinIO Contribution Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
 
 ``MinIO`` community welcomes your contribution. To make the process as seamless as possible, we recommend you read this contribution guide.
 

Dockerfile

@@ -5,23 +5,24 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 ENV GOPATH /go
 ENV CGO_ENABLED 0
 ENV GO111MODULE on
-ENV GOPROXY https://proxy.golang.org
 
 RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio && cd minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
 
-FROM alpine:3.9
+FROM alpine:3.10
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
+COPY CREDITS /third_party/
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \

Dockerfile.arm.release

@@ -0,0 +1,41 @@
+FROM golang:1.13-alpine as builder
+
+WORKDIR /home
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+     apk add --no-cache git 'curl>7.61.0' && \
+     git clone https://github.com/minio/minio && \
+     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
+
+FROM arm32v7/alpine:3.10
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
+COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.arm64.release

@@ -0,0 +1,41 @@
+FROM golang:1.13-alpine as builder
+
+WORKDIR /home
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+     apk add --no-cache git 'curl>7.61.0' && \
+     git clone https://github.com/minio/minio && \
+     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
+
+FROM arm64v8/alpine:3.10
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
+COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.dev

@@ -1,14 +1,16 @@
-FROM alpine:3.9
+FROM alpine:3.10
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 COPY minio /usr/bin/
+COPY CREDITS /third_party/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \

Dockerfile.release

@@ -8,16 +8,18 @@ RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio
 
-FROM alpine:3.9
+FROM alpine:3.10
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \

Dockerfile.simpleci

@@ -34,11 +34,13 @@ USER ci
 RUN make
 RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
 RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
-RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
-RUN make verifiers
+RUN make test-race
 RUN make crosscompile
 RUN make verify
 
+## -- add healing tests
+RUN make verify-healing
+
 #-------------------------------------------------------------
 # Stage 2: Test Frontend
 #-------------------------------------------------------------

Makefile

@@ -22,7 +22,7 @@ getdeps:
 ifeq ($(GOARCH),s390x)
 	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && GO111MODULE=off go get honnef.co/go/tools/cmd/staticcheck)
 else
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2019.2.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2020.1.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
 endif
 	@which misspell 1>/dev/null || (echo "Installing misspell" && GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell)
 
@@ -64,20 +64,26 @@ test: verifiers build
 	@echo "Running unit tests"
 	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
+test-race: verifiers build
+	@echo "Running unit tests under -race"
+	@(env bash $(PWD)/buildscripts/race.sh)
+
 # Verify minio binary
 verify:
 	@echo "Verifying build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
-coverage: build
-	@echo "Running all coverage for minio"
-	@(env bash $(PWD)/buildscripts/go-coverage.sh)
+# Verify healing of disks with minio binary
+verify-healing:
+	@echo "Verify healing build with race"
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath  --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev
@@ -95,3 +101,4 @@ clean:
 	@rm -rvf minio
 	@rm -rvf build
 	@rm -rvf release
+	@rm -rvf .verify*

NOTICE

@@ -1,4 +1,4 @@
-MinIO Cloud Storage, (C) 2014,2015 MinIO, Inc.
+MinIO Cloud Storage, (C) 2014-2020 MinIO, Inc.
 
 This product includes software developed at MinIO, Inc.
 (https://min.io/).
@@ -6,4 +6,4 @@ This product includes software developed at MinIO, Inc.
 The MinIO project contains unmodified/modified subcomponents too with
 separate copyright notices and license terms. Your use of the source
 code for these subcomponents is subject to the terms and conditions
-of the following licenses.
+of Apache License Version 2.0

README.md

@@ -1,5 +1,5 @@
 # MinIO Quickstart Guide
-[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
+[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
 
 [![MinIO](https://raw.githubusercontent.com/minio/minio/master/.github/logo.svg?sanitize=true)](https://min.io)
 
@@ -21,7 +21,7 @@ docker run -p 9000:9000 minio/minio:edge server /data
 > NOTE: Docker will not display the default keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use default keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
-### Homebrew
+### Homebrew (recommended)
 Install minio packages using [Homebrew](http://brew.sh/)
 ```sh
 brew install minio/stable/minio
@@ -74,7 +74,7 @@ minio.exe server D:\Photos
 
 ## FreeBSD
 ### Port
-Install minio packages using [pkg](https://github.com/freebsd/pkg)
+Install minio packages using [pkg](https://github.com/freebsd/pkg), MinIO doesn't officially build FreeBSD binaries but is maintained by FreeBSD upstream [here](https://www.freshports.org/www/minio).
 
 ```sh
 pkg install minio

README_zh_CN.md

@@ -1,4 +1,4 @@
-# MinIO Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
+# MinIO Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
 
 MinIO 是一个基于Apache License v2.0开源协议的对象存储服务。它兼容亚马逊S3云存储服务接口，非常适合于存储大容量非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。
 

SECURITY.md

@@ -8,12 +8,12 @@ Whenever there is a security update you just need to upgrade to the latest versi
 ## Reporting a Vulnerability
 
 All security bugs in [minio/minio](https://github,com/minio/minio) (or other minio/* repositories)
-should be reported by email to security@min.io. Your email will be acknowledged within 48 hours, 
-and you'll receive a more detailed response to your email within 72 hours indicating the next steps 
-in handling your report. 
+should be reported by email to security@min.io. Your email will be acknowledged within 48 hours,
+and you'll receive a more detailed response to your email within 72 hours indicating the next steps
+in handling your report.
 
-Please, provide a detailed explanation of the issue. In particular, outline the type of the security 
-issue (DoS, authentication bypass, information disclouse, ...) and the assumptions you're making (e.g. do 
+Please, provide a detailed explanation of the issue. In particular, outline the type of the security
+issue (DoS, authentication bypass, information disclose, ...) and the assumptions you're making (e.g. do
 you need access credentials for a successful exploit).
 
 If you have not received a reply to your email within 48 hours or you have not heard from the security team
@@ -28,14 +28,14 @@ MinIO uses the following disclosure process:
 
 1. Once the security report is received one member of the security team tries to verify and reproduce
    the issue and determines the impact it has.
-2. A member of the security team will respond and either confrim or reject the security report.
-   If the report is rejected the repsonse explains why.
+2. A member of the security team will respond and either confirm or reject the security report.
+   If the report is rejected the response explains why.
 3. Code is audited to find any potential similar problems.
-4. Fixes are prepared for the latest release. 
+4. Fixes are prepared for the latest release.
 5. On the date that the fixes are applied a security advisory will be published on https://blog.min.io.
    Please inform us in your report email whether MinIO should mention your contribution w.r.t. fixing
    the security issue. By default MinIO will **not** publish this information to protect your privacy.
-   
+
 This process can take some time, especially when coordination is required with maintainers of other projects.
-Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we 
-follow the process described above to ensure that disclosures are handled consistently. 
+Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we
+follow the process described above to ensure that disclosures are handled consistently.

browser/app/js/browser/Login.js

@@ -148,7 +148,8 @@ export class Login extends React.Component {
                   <OpenIDLoginButton
                     className="btn openid-btn"
                     clientId={this.state.clientId}
-                    authorizationEndpoint={this.state.discoveryDoc.authorization_endpoint}
+                    authEp={this.state.discoveryDoc.authorization_endpoint}
+                    authScopes={this.state.discoveryDoc.scopes_supported}
                   >
                     Log in with OpenID
                   </OpenIDLoginButton>

browser/app/js/browser/OpenIDLogin.js

@@ -66,6 +66,7 @@ export class OpenIDLogin extends React.Component {
 
       const authURL = buildOpenIDAuthURL(
         this.state.discoveryDoc.authorization_endpoint,
+        this.state.discoveryDoc.scopes_supported,
         redirectURI,
         this.state.clientID,
         nonce

browser/app/js/browser/OpenIDLoginButton.js

@@ -27,7 +27,7 @@ export class OpenIDLoginButton extends React.Component {
 
   handleClick(event) {
     event.stopPropagation()
-    const { authorizationEndpoint, clientId } = this.props
+    const { authEp, authScopes, clientId } = this.props
 
     let redirectURI = window.location.href.split("#")[0]
     if (redirectURI.endsWith('/')) {
@@ -40,7 +40,7 @@ export class OpenIDLoginButton extends React.Component {
     const nonce = getRandomString(16)
     storage.setItem(OPEN_ID_NONCE_KEY, nonce)
 
-    const authURL = buildOpenIDAuthURL(authorizationEndpoint, redirectURI, clientId, nonce)
+    const authURL = buildOpenIDAuthURL(authEp, authScopes, redirectURI, clientId, nonce)
     window.location = authURL
   }
 

browser/app/js/browser/utils.js

@@ -16,14 +16,13 @@
 
 export const OPEN_ID_NONCE_KEY = 'openIDKey'
 
-export const buildOpenIDAuthURL = (authorizationEndpoint, redirectURI, clientID, nonce) => {
+export const buildOpenIDAuthURL = (authEp, authScopes, redirectURI, clientID, nonce) => {
   const params = new URLSearchParams()
   params.set("response_type", "id_token")
-  params.set("scope", "openid")
+  params.set("scope", authScopes.join(" "))
   params.set("client_id", clientID)
   params.set("redirect_uri", redirectURI)
   params.set("nonce", nonce)
 
-  return `${authorizationEndpoint}?${params.toString()}`
+  return `${authEp}?${params.toString()}`
 }
-

browser/app/js/mime.js

@@ -14,30 +14,67 @@
  * limitations under the License.
  */
 
-import mimedb from 'mime-types'
+import mimedb from "mime-types"
 
 const isFolder = (name, contentType) => {
-  if (name.endsWith('/')) return true
+  if (name.endsWith("/")) return true
   return false
 }
 
 const isPdf = (name, contentType) => {
-  if (contentType === 'application/pdf') return true
+  if (contentType === "application/pdf") return true
+  return false
+}
+const isImage = (name, contentType) => {
+  if (
+    contentType === "image/jpeg" ||
+    contentType === "image/gif" ||
+    contentType === "image/x-icon" ||
+    contentType === "image/png" ||
+    contentType === "image/svg+xml" ||
+    contentType === "image/tiff" ||
+    contentType === "image/webp"
+  )
+    return true
   return false
 }
 
 const isZip = (name, contentType) => {
-  if (!contentType || !contentType.includes('/')) return false
-  if (contentType.split('/')[1].includes('zip')) return true
+  if (!contentType || !contentType.includes("/")) return false
+  if (contentType.split("/")[1].includes("zip")) return true
   return false
 }
 
 const isCode = (name, contentType) => {
-  const codeExt = ['c', 'cpp', 'go', 'py', 'java', 'rb', 'js', 'pl', 'fs',
-    'php', 'css', 'less', 'scss', 'coffee', 'net', 'html',
-    'rs', 'exs', 'scala', 'hs', 'clj', 'el', 'scm', 'lisp',
-    'asp', 'aspx']
-  const ext = name.split('.').reverse()[0]
+  const codeExt = [
+    "c",
+    "cpp",
+    "go",
+    "py",
+    "java",
+    "rb",
+    "js",
+    "pl",
+    "fs",
+    "php",
+    "css",
+    "less",
+    "scss",
+    "coffee",
+    "net",
+    "html",
+    "rs",
+    "exs",
+    "scala",
+    "hs",
+    "clj",
+    "el",
+    "scm",
+    "lisp",
+    "asp",
+    "aspx",
+  ]
+  const ext = name.split(".").reverse()[0]
   for (var i in codeExt) {
     if (ext === codeExt[i]) return true
   }
@@ -45,9 +82,9 @@ const isCode = (name, contentType) => {
 }
 
 const isExcel = (name, contentType) => {
-  if (!contentType || !contentType.includes('/')) return false
-  const types = ['excel', 'spreadsheet']
-  const subType = contentType.split('/')[1]
+  if (!contentType || !contentType.includes("/")) return false
+  const types = ["excel", "spreadsheet"]
+  const subType = contentType.split("/")[1]
   for (var i in types) {
     if (subType.includes(types[i])) return true
   }
@@ -55,9 +92,9 @@ const isExcel = (name, contentType) => {
 }
 
 const isDoc = (name, contentType) => {
-  if (!contentType || !contentType.includes('/')) return false
-  const types = ['word', '.document']
-  const subType = contentType.split('/')[1]
+  if (!contentType || !contentType.includes("/")) return false
+  const types = ["word", ".document"]
+  const subType = contentType.split("/")[1]
   for (var i in types) {
     if (subType.includes(types[i])) return true
   }
@@ -65,9 +102,9 @@ const isDoc = (name, contentType) => {
 }
 
 const isPresentation = (name, contentType) => {
-  if (!contentType || !contentType.includes('/')) return false
-  var types = ['powerpoint', 'presentation']
-  const subType = contentType.split('/')[1]
+  if (!contentType || !contentType.includes("/")) return false
+  var types = ["powerpoint", "presentation"]
+  const subType = contentType.split("/")[1]
   for (var i in types) {
     if (subType.includes(types[i])) return true
   }
@@ -76,31 +113,32 @@ const isPresentation = (name, contentType) => {
 
 const typeToIcon = (type) => {
   return (name, contentType) => {
-    if (!contentType || !contentType.includes('/')) return false
-    if (contentType.split('/')[0] === type) return true
+    if (!contentType || !contentType.includes("/")) return false
+    if (contentType.split("/")[0] === type) return true
     return false
   }
 }
 
 export const getDataType = (name, contentType) => {
   if (contentType === "") {
-    contentType = mimedb.lookup(name) || 'application/octet-stream'
+    contentType = mimedb.lookup(name) || "application/octet-stream"
   }
   const check = [
-    ['folder', isFolder],
-    ['code', isCode],
-    ['audio', typeToIcon('audio')],
-    ['image', typeToIcon('image')],
-    ['video', typeToIcon('video')],
-    ['text', typeToIcon('text')],
-    ['pdf', isPdf],
-    ['zip', isZip],
-    ['excel', isExcel],
-    ['doc', isDoc],
-    ['presentation', isPresentation]
+    ["folder", isFolder],
+    ["code", isCode],
+    ["audio", typeToIcon("audio")],
+    ["image", typeToIcon("image")],
+    ["video", typeToIcon("video")],
+    ["text", typeToIcon("text")],
+    ["pdf", isPdf],
+    ["image", isImage],
+    ["zip", isZip],
+    ["excel", isExcel],
+    ["doc", isDoc],
+    ["presentation", isPresentation],
   ]
   for (var i in check) {
     if (check[i][1](name, contentType)) return check[i][0]
   }
-  return 'other'
+  return "other"
 }

browser/app/js/objects/ObjectActions.js

@@ -19,18 +19,22 @@ import { connect } from "react-redux"
 import { Dropdown } from "react-bootstrap"
 import ShareObjectModal from "./ShareObjectModal"
 import DeleteObjectConfirmModal from "./DeleteObjectConfirmModal"
+import PreviewObjectModal from "./PreviewObjectModal"
+
 import * as objectsActions from "./actions"
+import { getDataType } from "../mime.js"
 import {
   SHARE_OBJECT_EXPIRY_DAYS,
   SHARE_OBJECT_EXPIRY_HOURS,
-  SHARE_OBJECT_EXPIRY_MINUTES
+  SHARE_OBJECT_EXPIRY_MINUTES,
 } from "../constants"
 
 export class ObjectActions extends React.Component {
   constructor(props) {
     super(props)
     this.state = {
-      showDeleteConfirmation: false
+      showDeleteConfirmation: false,
+      showPreview: false,
     }
   }
   shareObject(e) {
@@ -53,7 +57,20 @@ export class ObjectActions extends React.Component {
   }
   hideDeleteConfirmModal() {
     this.setState({
-      showDeleteConfirmation: false
+      showDeleteConfirmation: false,
+    })
+  }
+  getObjectURL(objectname, callback) {
+    const { getObjectURL } = this.props
+    getObjectURL(objectname, callback)
+  }
+  showPreviewModal(e) {
+    e.preventDefault()
+    this.setState({ showPreview: true })
+  }
+  hidePreviewModal() {
+    this.setState({
+      showPreview: false,
     })
   }
   render() {
@@ -69,6 +86,15 @@ export class ObjectActions extends React.Component {
           >
             <i className="fas fa-share-alt" />
           </a>
+          {getDataType(object.name, object.contentType) == "image" && (
+            <a
+              href=""
+              className="fiad-action"
+              onClick={this.showPreviewModal.bind(this)}
+            >
+              <i className="far fa-file-image" />
+            </a>
+          )}
           <a
             href=""
             className="fiad-action"
@@ -77,14 +103,22 @@ export class ObjectActions extends React.Component {
             <i className="fas fa-trash-alt" />
           </a>
         </Dropdown.Menu>
-        {(showShareObjectModal && shareObjectName === object.name) &&
-          <ShareObjectModal object={object} />}
+        {showShareObjectModal && shareObjectName === object.name && (
+          <ShareObjectModal object={object} />
+        )}
         {this.state.showDeleteConfirmation && (
           <DeleteObjectConfirmModal
             deleteObject={this.deleteObject.bind(this)}
             hideDeleteConfirmModal={this.hideDeleteConfirmModal.bind(this)}
           />
         )}
+        {this.state.showPreview && (
+          <PreviewObjectModal
+            object={object}
+            hidePreviewModal={this.hidePreviewModal.bind(this)}
+            getObjectURL={this.getObjectURL.bind(this)}
+          />
+        )}
       </Dropdown>
     )
   }
@@ -94,15 +128,17 @@ const mapStateToProps = (state, ownProps) => {
   return {
     object: ownProps.object,
     showShareObjectModal: state.objects.shareObject.show,
-    shareObjectName: state.objects.shareObject.object
+    shareObjectName: state.objects.shareObject.object,
   }
 }
 
-const mapDispatchToProps = dispatch => {
+const mapDispatchToProps = (dispatch) => {
   return {
     shareObject: (object, days, hours, minutes) =>
       dispatch(objectsActions.shareObject(object, days, hours, minutes)),
-    deleteObject: object => dispatch(objectsActions.deleteObject(object))
+    deleteObject: (object) => dispatch(objectsActions.deleteObject(object)),
+    getObjectURL: (object, callback) =>
+      dispatch(objectsActions.getObjectURL(object, callback)),
   }
 }
 

browser/app/js/objects/PreviewObjectModal.js

@@ -0,0 +1,65 @@
+/*
+ * MinIO Cloud Storage (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { Modal, ModalHeader, ModalBody } from "react-bootstrap"
+
+class PreviewObjectModal extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      url: "",
+    }
+    props.getObjectURL(props.object.name, (url) => {
+      this.setState({
+        url: url,
+      })
+    })
+  }
+
+  render() {
+    const { hidePreviewModal } = this.props
+    return (
+      <Modal
+        show={true}
+        animation={false}
+        onHide={hidePreviewModal}
+        bsSize="large"
+      >
+        <ModalHeader>Preview</ModalHeader>
+        <ModalBody>
+          <div className="input-group">
+            {this.state.url && (
+              <img
+                alt="Image broken"
+                src={this.state.url}
+                style={{ display: "block", width: "100%" }}
+              />
+            )}
+          </div>
+        </ModalBody>
+        <div className="modal-footer">
+          {
+            <button className="btn btn-link" onClick={hidePreviewModal}>
+              Cancel
+            </button>
+          }
+        </div>
+      </Modal>
+    )
+  }
+}
+export default PreviewObjectModal

browser/app/js/objects/__tests__/ObjectActions.test.js

@@ -66,6 +66,49 @@ describe("ObjectActions", () => {
     expect(deleteObject).toHaveBeenCalledWith("obj1")
   })
 
+
+
+
+  it("should show PreviewObjectModal when preview action is clicked", () => {
+    const wrapper = shallow(
+      <ObjectActions 
+      object={{ name: "obj1", contentType: "image/jpeg"}} 
+      currentPrefix={"pre1/"} />
+    )
+    wrapper
+      .find("a")
+      .at(1)
+      .simulate("click", { preventDefault: jest.fn() })
+    expect(wrapper.state("showPreview")).toBeTruthy()
+    expect(wrapper.find("PreviewObjectModal").length).toBe(1)
+  })
+
+  it("should hide PreviewObjectModal when cancel button is clicked", () => {
+    const wrapper = shallow(
+      <ObjectActions 
+        object={{ name: "obj1" , contentType: "image/jpeg"}}
+        currentPrefix={"pre1/"} />
+    )
+    wrapper
+      .find("a")
+      .at(1)
+      .simulate("click", { preventDefault: jest.fn() })
+    wrapper.find("PreviewObjectModal").prop("hidePreviewModal")()
+    wrapper.update()
+    expect(wrapper.state("showPreview")).toBeFalsy()
+    expect(wrapper.find("PreviewObjectModal").length).toBe(0)
+  })
+  it("should not show PreviewObjectModal when preview action is clicked if object is not an image", () => {
+    const wrapper = shallow(
+      <ObjectActions 
+      object={{ name: "obj1"}} 
+      currentPrefix={"pre1/"} />
+    )
+    expect(wrapper
+      .find("a")
+      .length).toBe(2) // find only the other 2
+  })
+
   it("should call shareObject with object and expiry", () => {
     const shareObject = jest.fn()
     const wrapper = shallow(

browser/app/js/objects/actions.js

@@ -19,7 +19,7 @@ import history from "../history"
 import {
   sortObjectsByName,
   sortObjectsBySize,
-  sortObjectsByDate
+  sortObjectsByDate,
 } from "../utils"
 import { getCurrentBucket } from "../buckets/selectors"
 import { getCurrentPrefix, getCheckedList } from "./selectors"
@@ -31,7 +31,7 @@ import {
   SORT_BY_SIZE,
   SORT_BY_LAST_MODIFIED,
   SORT_ORDER_ASC,
-  SORT_ORDER_DESC
+  SORT_ORDER_DESC,
 } from "../constants"
 
 export const SET_LIST = "objects/SET_LIST"
@@ -48,35 +48,35 @@ export const CHECKED_LIST_REMOVE = "objects/CHECKED_LIST_REMOVE"
 export const CHECKED_LIST_RESET = "objects/CHECKED_LIST_RESET"
 export const SET_LIST_LOADING = "objects/SET_LIST_LOADING"
 
-export const setList = objects => ({
+export const setList = (objects) => ({
   type: SET_LIST,
-  objects
+  objects,
 })
 
 export const resetList = () => ({
-  type: RESET_LIST
+  type: RESET_LIST,
 })
 
-export const setListLoading = listLoading => ({
+export const setListLoading = (listLoading) => ({
   type: SET_LIST_LOADING,
-  listLoading
+  listLoading,
 })
 
 export const fetchObjects = () => {
-  return function(dispatch, getState) {
+  return function (dispatch, getState) {
     dispatch(resetList())
     const {
       buckets: { currentBucket },
-      objects: { currentPrefix }
+      objects: { currentPrefix },
     } = getState()
     if (currentBucket) {
       dispatch(setListLoading(true))
       return web
         .ListObjects({
           bucketName: currentBucket,
-          prefix: currentPrefix
+          prefix: currentPrefix,
         })
-        .then(res => {
+        .then((res) => {
           // we need to check if the bucket name and prefix are the same as
           // when the request was made before updating the displayed objects
           if (
@@ -85,10 +85,10 @@ export const fetchObjects = () => {
           ) {
             let objects = []
             if (res.objects) {
-              objects = res.objects.map(object => {
+              objects = res.objects.map((object) => {
                 return {
                   ...object,
-                  name: object.name.replace(currentPrefix, "")
+                  name: object.name.replace(currentPrefix, ""),
                 }
               })
             }
@@ -104,13 +104,13 @@ export const fetchObjects = () => {
             dispatch(setListLoading(false))
           }
         })
-        .catch(err => {
+        .catch((err) => {
           if (web.LoggedIn()) {
             dispatch(
               alertActions.set({
                 type: "danger",
                 message: err.message,
-                autoClear: true
+                autoClear: true,
               })
             )
             dispatch(resetList())
@@ -123,8 +123,8 @@ export const fetchObjects = () => {
   }
 }
 
-export const sortObjects = sortBy => {
-  return function(dispatch, getState) {
+export const sortObjects = (sortBy) => {
+  return function (dispatch, getState) {
     const { objects } = getState()
     let sortOrder = SORT_ORDER_ASC
     // Reverse sort order if the list is already sorted on same field
@@ -149,18 +149,18 @@ const sortObjectsList = (list, sortBy, sortOrder) => {
   }
 }
 
-export const setSortBy = sortBy => ({
+export const setSortBy = (sortBy) => ({
   type: SET_SORT_BY,
-  sortBy
+  sortBy,
 })
 
-export const setSortOrder = sortOrder => ({
+export const setSortOrder = (sortOrder) => ({
   type: SET_SORT_ORDER,
-  sortOrder
+  sortOrder,
 })
 
-export const selectPrefix = prefix => {
-  return function(dispatch, getState) {
+export const selectPrefix = (prefix) => {
+  return function (dispatch, getState) {
     dispatch(setCurrentPrefix(prefix))
     dispatch(fetchObjects())
     dispatch(resetCheckedList())
@@ -169,49 +169,49 @@ export const selectPrefix = prefix => {
   }
 }
 
-export const setCurrentPrefix = prefix => {
+export const setCurrentPrefix = (prefix) => {
   return {
     type: SET_CURRENT_PREFIX,
-    prefix
+    prefix,
   }
 }
 
-export const setPrefixWritable = prefixWritable => ({
+export const setPrefixWritable = (prefixWritable) => ({
   type: SET_PREFIX_WRITABLE,
-  prefixWritable
+  prefixWritable,
 })
 
-export const deleteObject = object => {
-  return function(dispatch, getState) {
+export const deleteObject = (object) => {
+  return function (dispatch, getState) {
     const currentBucket = getCurrentBucket(getState())
     const currentPrefix = getCurrentPrefix(getState())
     const objectName = `${currentPrefix}${object}`
     return web
       .RemoveObject({
         bucketName: currentBucket,
-        objects: [objectName]
+        objects: [objectName],
       })
       .then(() => {
         dispatch(removeObject(object))
       })
-      .catch(e => {
+      .catch((e) => {
         dispatch(
           alertActions.set({
             type: "danger",
-            message: e.message
+            message: e.message,
           })
         )
       })
   }
 }
 
-export const removeObject = object => ({
+export const removeObject = (object) => ({
   type: REMOVE,
-  object
+  object,
 })
 
 export const deleteCheckedObjects = () => {
-  return function(dispatch, getState) {
+  return function (dispatch, getState) {
     const checkedObjects = getCheckedList(getState())
     for (let i = 0; i < checkedObjects.length; i++) {
       dispatch(deleteObject(checkedObjects[i]))
@@ -221,7 +221,7 @@ export const deleteCheckedObjects = () => {
 }
 
 export const shareObject = (object, days, hours, minutes) => {
-  return function(dispatch, getState) {
+  return function (dispatch, getState) {
     const currentBucket = getCurrentBucket(getState())
     const currentPrefix = getCurrentPrefix(getState())
     const objectName = `${currentPrefix}${object}`
@@ -232,22 +232,22 @@ export const shareObject = (object, days, hours, minutes) => {
           host: location.host,
           bucket: currentBucket,
           object: objectName,
-          expiry: expiry
+          expiry: expiry,
         })
-        .then(obj => {
+        .then((obj) => {
           dispatch(showShareObject(object, obj.url))
           dispatch(
             alertActions.set({
               type: "success",
-              message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`
+              message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`,
             })
           )
         })
-        .catch(err => {
+        .catch((err) => {
           dispatch(
             alertActions.set({
               type: "danger",
-              message: err.message
+              message: err.message,
             })
           )
         })
@@ -265,7 +265,7 @@ export const shareObject = (object, days, hours, minutes) => {
       dispatch(
         alertActions.set({
           type: "success",
-          message: `Object shared.`
+          message: `Object shared.`,
         })
       )
     }
@@ -276,18 +276,17 @@ export const showShareObject = (object, url) => ({
   type: SET_SHARE_OBJECT,
   show: true,
   object,
-  url
+  url,
 })
 
 export const hideShareObject = (object, url) => ({
   type: SET_SHARE_OBJECT,
   show: false,
   object: "",
-  url: ""
+  url: "",
 })
-
-export const downloadObject = object => {
-  return function(dispatch, getState) {
+export const getObjectURL = (object, callback) => {
+  return function (dispatch, getState) {
     const currentBucket = getCurrentBucket(getState())
     const currentPrefix = getCurrentPrefix(getState())
     const objectName = `${currentPrefix}${object}`
@@ -295,52 +294,73 @@ export const downloadObject = object => {
     if (web.LoggedIn()) {
       return web
         .CreateURLToken()
-        .then(res => {
-          const url = `${
-            window.location.origin
-          }${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=${
-            res.token
-          }`
-          window.location = url
+        .then((res) => {
+          const url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=${res.token}`
+          callback(url)
         })
-        .catch(err => {
+        .catch((err) => {
           dispatch(
             alertActions.set({
               type: "danger",
-              message: err.message
+              message: err.message,
             })
           )
         })
     } else {
-      const url = `${
-        window.location.origin
-      }${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=`
+      const url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=`
+      callback(url)
+    }
+  }
+}
+export const downloadObject = (object) => {
+  return function (dispatch, getState) {
+    const currentBucket = getCurrentBucket(getState())
+    const currentPrefix = getCurrentPrefix(getState())
+    const objectName = `${currentPrefix}${object}`
+    const encObjectName = encodeURI(objectName)
+    if (web.LoggedIn()) {
+      return web
+        .CreateURLToken()
+        .then((res) => {
+          const url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=${res.token}`
+          window.location = url
+        })
+        .catch((err) => {
+          dispatch(
+            alertActions.set({
+              type: "danger",
+              message: err.message,
+            })
+          )
+        })
+    } else {
+      const url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=`
       window.location = url
     }
   }
 }
 
-export const checkObject = object => ({
+export const checkObject = (object) => ({
   type: CHECKED_LIST_ADD,
-  object
+  object,
 })
 
-export const uncheckObject = object => ({
+export const uncheckObject = (object) => ({
   type: CHECKED_LIST_REMOVE,
-  object
+  object,
 })
 
 export const resetCheckedList = () => ({
-  type: CHECKED_LIST_RESET
+  type: CHECKED_LIST_RESET,
 })
 
 export const downloadCheckedObjects = () => {
-  return function(dispatch, getState) {
+  return function (dispatch, getState) {
     const state = getState()
     const req = {
       bucketName: getCurrentBucket(state),
       prefix: getCurrentPrefix(state),
-      objects: getCheckedList(state)
+      objects: getCheckedList(state),
     }
     if (!web.LoggedIn()) {
       const requestUrl = location.origin + "/minio/zip?token="
@@ -348,17 +368,15 @@ export const downloadCheckedObjects = () => {
     } else {
       return web
         .CreateURLToken()
-        .then(res => {
-          const requestUrl = `${
-            location.origin
-          }${minioBrowserPrefix}/zip?token=${res.token}`
+        .then((res) => {
+          const requestUrl = `${location.origin}${minioBrowserPrefix}/zip?token=${res.token}`
           downloadZip(requestUrl, req, dispatch)
         })
-        .catch(err =>
+        .catch((err) =>
           dispatch(
             alertActions.set({
               type: "danger",
-              message: err.message
+              message: err.message,
             })
           )
         )
@@ -374,11 +392,11 @@ const downloadZip = (url, req, dispatch) => {
   xhr.open("POST", url, true)
   xhr.responseType = "blob"
 
-  xhr.onload = function(e) {
+  xhr.onload = function (e) {
     if (this.status == 200) {
       dispatch(resetCheckedList())
       var blob = new Blob([this.response], {
-        type: "octet/stream"
+        type: "octet/stream",
       })
       var blobUrl = window.URL.createObjectURL(blob)
       var separator = req.prefix.length > 1 ? "-" : ""

browser/staticcheck.conf

@@ -0,0 +1 @@
+checks = ["all", "-ST1005", "-ST1000", "-SA4000", "-SA9004", "-SA1019", "-SA1008", "-U1000", "-ST1003", "-ST1018"]

buildscripts/benchcmp.sh

@@ -1,67 +0,0 @@
-#!/bin/bash          
-
-# usage: ./benchcmp.sh <commit-sha1> <commit-sha2>
-# Exit on any non zero return value on execution of a command. 
-set -e 
-
-# path of benchcmp. 
-benchcmp=${GOPATH}/bin/benchcmp
-
-# function which runs the benchmark comparison. 
-RunBenchCmp () {
-	# Path for storing output of benchmark at commit 1. 
-	commit1Bench=/tmp/minio-$1.bench
-	# Path for storing output of benchmark at commit 2. 
-	commit2Bench=/tmp/minio-$2.bench
-	# switch to commit $1. 
-	git checkout $1
-	# Check if the benchmark results for given commit 1 already exists. 
-	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
-	if [[ ! -f $commit1Bench  ]]
-	then
-
-		echo "Running benchmarks at $1"
-		go test -run=NONE -bench=. | tee $commit1Bench 
-	fi
-	# get back to the commit from which it was started. 
-	git checkout -
-	echo "Checking into commit $2"
-	# switch to commit $2 
-	git checkout $2
-	# Check if the benchmark results for given commit 2 already exists. 
-	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
-	if [[ ! -f $commit2Bench  ]]
-	then
-	# Running benchmarks at $2.
-		echo "Running benchmarks at $2"
-		go test -run=NONE -bench=. | tee $commit2Bench 
-	fi
-	
-	# get back to the commit from which it was started. 
-	git checkout - 
-	# Comparing the benchmarks.
-	echo "Running benchmark comparison between $1 and $2 ..."
-	$benchcmp $commit1Bench $commit2Bench 
-	echo "Done."
-}
-
-# check if 2 commit SHA's of snapshots of code for which benchmp has to be done is provided. 
-if [ ! $# -eq 2 ]
-then
-	# exit if commit SHA's are not provided. 
-	echo $#
-	echo "Need Commit SHA's of 2 snapshots to be supplied to run benchmark comparison."
-	exit 1
-fi
-
-# check if benchcmp exists. 
-if [[ -x "$benchcmp" ]]
-then
-	RunBenchCmp $1 $2
-else
-	# install benchcmp if doesnt't exist. 
-	echo "fetching Benchcmp..."
-	go get -u golang.org/x/tools/cmd/benchcmp
-	echo "Done."
-	RunBenchCmp $1 $2
-fi

buildscripts/cross-compile.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
 
+set -e
 # Enable tracing if set.
-[ -n "$BASH_XTRACEFD" ] && set -ex
+[ -n "$BASH_XTRACEFD" ] && set -x
 
 function _init() {
     ## All binaries are static make sure to disable CGO.
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64"
+    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386"
 }
 
 function _build() {
@@ -19,11 +20,11 @@ function _build() {
     package=$(go list -f '{{.ImportPath}}')
     printf -- "--> %15s:%s\n" "${osarch}" "${package}"
 
-    # Go build to build the binary.
+    # go build -trimpath to build the binary.
     export GOOS=$os
     export GOARCH=$arch
     export GO111MODULE=on
-    go build -tags kqueue -o /dev/null
+    go build -trimpath -tags kqueue -o /dev/null
 }
 
 function main() {

buildscripts/gateway-tests.sh

@@ -48,8 +48,8 @@ function main()
     SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \
                    SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh \
                    aws-sdk-go aws-sdk-java aws-sdk-php aws-sdk-ruby awscli \
-                   healthcheck mc minio-dotnet minio-go minio-java minio-py \
-                   s3cmd security
+                   healthcheck mc minio-dotnet minio-java minio-js \
+                   minio-py s3cmd s3select security
     rv=$?
 
     kill "$sr_pid"

buildscripts/go-coverage.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...

buildscripts/race.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+
+for d in $(go list ./... | grep -v browser); do
+    CGO_ENABLED=1 go test -v -race --timeout 50m "$d"
+done

buildscripts/verify-healing.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+#
+# MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+set -E
+set -o pipefail
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+function start_minio_3_node() {
+    declare -a minio_pids
+    declare -a ARGS
+    export MINIO_ACCESS_KEY=minio
+    export MINIO_SECRET_KEY=minio123
+
+    start_port=$(shuf -i 10000-65000 -n 1)
+    for i in $(seq 1 3); do
+        ARGS+=("http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/")
+    done
+
+    "${MINIO[@]}" --address ":$[$start_port+1]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
+    minio_pids[0]=$!
+    disown "${minio_pids[0]}"
+
+    "${MINIO[@]}" --address ":$[$start_port+2]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
+    minio_pids[1]=$!
+    disown "${minio_pids[1]}"
+
+    "${MINIO[@]}" --address ":$[$start_port+3]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
+    minio_pids[2]=$!
+    disown "${minio_pids[2]}"
+
+    sleep "$1"
+    for pid in "${minio_pids[@]}"; do
+        if ! kill "$pid"; then
+            for i in $(seq 1 3); do
+                echo "server$i log:"
+                cat "${WORK_DIR}/dist-minio-server$i.log"
+            done
+            echo "FAILED"
+            purge "$WORK_DIR"
+            exit 1
+        fi
+        # forcibly killing, to proceed further properly.
+        kill -9 "$pid"
+        sleep 1 # wait 1sec per pid
+    done
+}
+
+
+function check_online() {
+    if grep -q 'Server switching to safe mode' ${WORK_DIR}/dist-minio-*.log; then
+        echo "1"
+    fi
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+function __init__()
+{
+    echo "Initializing environment"
+    mkdir -p "$WORK_DIR"
+    mkdir -p "$MINIO_CONFIG_DIR"
+
+    ## version is purposefully set to '3' for minio to migrate configuration file
+    echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
+}
+
+function perform_test() {
+    start_minio_3_node 60
+
+    echo "Testing Distributed Erasure setup healing of drives"
+    echo "Remove the contents of the disks belonging to '${1}' erasure set"
+
+    rm -rf ${WORK_DIR}/${1}/*/
+
+    start_minio_3_node 60
+
+    rv=$(check_online)
+    if [ "$rv" == "1" ]; then
+        pkill -9 minio
+        for i in $(seq 1 3); do
+            echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-server$i.log"
+        done
+        echo "FAILED"
+        purge "$WORK_DIR"
+        exit 1
+    fi
+}
+
+function main()
+{
+    perform_test "2"
+    perform_test "1"
+    perform_test "3"
+}
+
+( __init__ "$@" && main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

cmd/acl-handlers.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,11 +18,14 @@ package cmd
 
 import (
 	"encoding/xml"
+	"io"
 	"net/http"
+	"net/url"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 // Data types used for returning dummy access control
@@ -50,6 +53,71 @@ type accessControlPolicy struct {
 	} `xml:"AccessControlList"`
 }
 
+// PutBucketACLHandler - PUT Bucket ACL
+// -----------------
+// This operation uses the ACL subresource
+// to set ACL for a bucket, this is a dummy call
+// only responds success if the ACL is private.
+func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketACL")
+
+	defer logger.AuditLog(w, r, "PutBucketACL", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow putBucketACL if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Before proceeding validate if bucket exists.
+	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	aclHeader := r.Header.Get(xhttp.AmzACL)
+	if aclHeader == "" {
+		acl := &accessControlPolicy{}
+		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
+			if err == io.EOF {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingSecurityHeader),
+					r.URL, guessIsBrowserReq(r))
+				return
+			}
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if len(acl.AccessControlList.Grants) == 0 {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
+	if aclHeader != "" && aclHeader != "private" {
+		writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
 // GetBucketACLHandler - GET Bucket ACL
 // -----------------
 // This operation uses the ACL
@@ -99,6 +167,71 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 	w.(http.Flusher).Flush()
 }
 
+// PutObjectACLHandler - PUT Object ACL
+// -----------------
+// This operation uses the ACL subresource
+// to set ACL for a bucket, this is a dummy call
+// only responds success if the ACL is private.
+func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutObjectACL")
+
+	defer logger.AuditLog(w, r, "PutObjectACL", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object, err := url.PathUnescape(vars["object"])
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow putObjectACL if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Before proceeding validate if object exists.
+	_, err = objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	aclHeader := r.Header.Get(xhttp.AmzACL)
+	if aclHeader == "" {
+		acl := &accessControlPolicy{}
+		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if len(acl.AccessControlList.Grants) == 0 {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
+	if aclHeader != "" && aclHeader != "private" {
+		writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
 // GetObjectACLHandler - GET Object ACL
 // -----------------
 // This operation uses the ACL
@@ -110,7 +243,11 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
-	object := vars["object"]
+	object, err := url.PathUnescape(vars["object"])
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -126,7 +263,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if object exists.
-	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	_, err = objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/admin-handlers-config-kv.go

@@ -57,7 +57,7 @@ func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *htt
 	return objectAPI
 }
 
-// DelConfigKVHandler - DELETE /minio/admin/v2/del-config-kv
+// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DelConfigKVHandler")
 
@@ -103,7 +103,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	}
 }
 
-// SetConfigKVHandler - PUT /minio/admin/v2/set-config-kv
+// SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfigKVHandler")
 
@@ -162,13 +162,13 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	// Make sure to write backend is encrypted
 	if globalConfigEncrypted {
-		saveConfig(context.Background(), objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
+		saveConfig(GlobalContext, objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 	}
 
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// GetConfigKVHandler - GET /minio/admin/v2/get-config-kv?key={key}
+// GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key}
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigKVHandler")
 
@@ -323,7 +323,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 	writeSuccessResponseJSON(w, econfigData)
 }
 
-// HelpConfigKVHandler - GET /minio/admin/v2/help-config-kv?subSys={subSys}&key={key}
+// HelpConfigKVHandler - GET /minio/admin/v3/help-config-kv?subSys={subSys}&key={key}
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HelpConfigKVHandler")
 
@@ -349,7 +349,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 	w.(http.Flusher).Flush()
 }
 
-// SetConfigHandler - PUT /minio/admin/v2/config
+// SetConfigHandler - PUT /minio/admin/v3/config
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfigHandler")
 
@@ -403,13 +403,13 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	// Make sure to write backend is encrypted
 	if globalConfigEncrypted {
-		saveConfig(context.Background(), objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
+		saveConfig(GlobalContext, objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 	}
 
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// GetConfigHandler - GET /minio/admin/v2/config
+// GetConfigHandler - GET /minio/admin/v3/config
 // Get config.json of this minio setup.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigHandler")

cmd/admin-handlers-users.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -51,7 +51,7 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 	return objectAPI, cred
 }
 
-// RemoveUser - DELETE /minio/admin/v2/remove-user?accessKey=<access_key>
+// RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey=<access_key>
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveUser")
 
@@ -93,15 +93,17 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// ListUsers - GET /minio/admin/v2/list-users
+// ListUsers - GET /minio/admin/v3/list-users
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListUsers")
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
+	password := cred.SecretKey
+
 	allCredentials, err := globalIAMSys.ListUsers()
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -114,7 +116,6 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	password := globalActiveCred.SecretKey
 	econfigData, err := madmin.EncryptData(password, data)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -124,7 +125,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseJSON(w, econfigData)
 }
 
-// GetUserInfo - GET /minio/admin/v2/user-info
+// GetUserInfo - GET /minio/admin/v3/user-info
 func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetUserInfo")
 
@@ -151,7 +152,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseJSON(w, data)
 }
 
-// UpdateGroupMembers - PUT /minio/admin/v2/update-group-members
+// UpdateGroupMembers - PUT /minio/admin/v3/update-group-members
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "UpdateGroupMembers")
 
@@ -194,7 +195,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 	}
 }
 
-// GetGroup - /minio/admin/v2/group?group=mygroup1
+// GetGroup - /minio/admin/v3/group?group=mygroup1
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetGroup")
 
@@ -221,7 +222,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseJSON(w, body)
 }
 
-// ListGroups - GET /minio/admin/v2/groups
+// ListGroups - GET /minio/admin/v3/groups
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListGroups")
 
@@ -245,7 +246,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseJSON(w, body)
 }
 
-// SetGroupStatus - PUT /minio/admin/v2/set-group-status?group=mygroup1&status=enabled
+// SetGroupStatus - PUT /minio/admin/v3/set-group-status?group=mygroup1&status=enabled
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetGroupStatus")
 
@@ -280,7 +281,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// SetUserStatus - PUT /minio/admin/v2/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
+// SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
 
@@ -319,7 +320,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// AddUser - PUT /minio/admin/v2/add-user?accessKey=<access_key>
+// AddUser - PUT /minio/admin/v3/add-user?accessKey=<access_key>
 func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddUser")
 
@@ -378,16 +379,137 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// InfoCannedPolicy - GET /minio/admin/v2/info-canned-policy?name={policyName}
-func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "InfoCannedPolicy")
+// AddServiceAccount - PUT /minio/admin/v3/add-service-account?parentUser=<parent_user_accesskey>
+func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddServiceAccount")
+
+	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := cred.SecretKey
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	var createReq madmin.AddServiceAccountReq
+	if err = json.Unmarshal(configBytes, &createReq); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	if createReq.Parent == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+
+	// Disallow creating service accounts for the root user
+	if createReq.Parent == globalActiveCred.AccessKey {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddServiceAccountInvalidArgument), r.URL)
+		return
+	}
+
+	creds, err := globalIAMSys.NewServiceAccount(ctx, createReq.Parent, createReq.Policy)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other Minio peers to reload user
+	for _, nerr := range globalNotificationSys.LoadUser(creds.AccessKey, false) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	var createResp = madmin.AddServiceAccountResp{
+		Credentials: creds,
+	}
+
+	data, err := json.Marshal(createResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// GetServiceAccount - GET /minio/admin/v3/get-service-account?accessKey=<access_key>
+func (a adminAPIHandlers) GetServiceAccount(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetServiceAccount")
+
+	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+
+	password := cred.SecretKey
+
+	creds, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(creds)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// InfoCannedPolicyV2 - GET /minio/admin/v2/info-canned-policy?name={policyName}
+func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "InfoCannedPolicyV2")
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	data, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	policy, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(policy)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -397,7 +519,58 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 	w.(http.Flusher).Flush()
 }
 
-// ListCannedPolicies - GET /minio/admin/v2/list-canned-policies
+// InfoCannedPolicy - GET /minio/admin/v3/info-canned-policy?name={policyName}
+func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "InfoCannedPolicy")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	policy, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	json.NewEncoder(w).Encode(policy)
+	w.(http.Flusher).Flush()
+}
+
+// ListCannedPoliciesV2 - GET /minio/admin/v2/list-canned-policies
+func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListCannedPoliciesV2")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	policies, err := globalIAMSys.ListPolicies()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	policyMap := make(map[string][]byte, len(policies))
+	for k, p := range policies {
+		var err error
+		policyMap[k], err = json.Marshal(p)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+	}
+	if err = json.NewEncoder(w).Encode(policyMap); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// ListCannedPolicies - GET /minio/admin/v3/list-canned-policies
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPolicies")
 
@@ -420,7 +593,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 	w.(http.Flusher).Flush()
 }
 
-// RemoveCannedPolicy - DELETE /minio/admin/v2/remove-canned-policy?name=<policy_name>
+// RemoveCannedPolicy - DELETE /minio/admin/v3/remove-canned-policy?name=<policy_name>
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveCannedPolicy")
 
@@ -452,7 +625,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	}
 }
 
-// AddCannedPolicy - PUT /minio/admin/v2/add-canned-policy?name=<policy_name>
+// AddCannedPolicy - PUT /minio/admin/v3/add-canned-policy?name=<policy_name>
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddCannedPolicy")
 
@@ -484,7 +657,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
@@ -508,7 +681,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	}
 }
 
-// SetPolicyForUserOrGroup - PUT /minio/admin/v2/set-policy?policy=xxx&user-or-group=?[&is-group]
+// SetPolicyForUserOrGroup - PUT /minio/admin/v3/set-policy?policy=xxx&user-or-group=?[&is-group]
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
 
@@ -530,7 +703,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 
 	if !isGroup {
 		ok, err := globalIAMSys.IsTempUser(entityName)
-		if err != nil {
+		if err != nil && err != errNoSuchUser {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}

cmd/admin-handlers.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"crypto/subtle"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -41,6 +42,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/event/target"
@@ -97,7 +99,7 @@ func updateServer(updateURL, sha256Hex string, latestReleaseTime time.Time) (us
 	return us, nil
 }
 
-// ServerUpdateHandler - POST /minio/admin/v2/update?updateURL={updateURL}
+// ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
 // ----------
 // updates all minio servers and restarts them gracefully.
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
@@ -140,9 +142,9 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		}
 
 		if runtime.GOOS == "windows" {
-			u.Path = path.Dir(u.Path) + "minio.exe"
+			u.Path = path.Dir(u.Path) + SlashSeparator + "minio.exe"
 		} else {
-			u.Path = path.Dir(u.Path) + "minio"
+			u.Path = path.Dir(u.Path) + SlashSeparator + "minio"
 		}
 
 		updateURL = u.String()
@@ -176,7 +178,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
-// ServiceActionHandler - POST /minio/admin/v2/service?action={action}
+// ServiceActionHandler - POST /minio/admin/v3/service?action={action}
 // ----------
 // restarts/stops minio server gracefully. In a distributed setup,
 func (a adminAPIHandlers) ServiceActionHandler(w http.ResponseWriter, r *http.Request) {
@@ -265,17 +267,17 @@ type ServerInfo struct {
 	Data  *ServerInfoData `json:"data"`
 }
 
-// StorageInfoHandler - GET /minio/admin/v2/storageinfo
+// StorageInfoHandler - GET /minio/admin/v3/storageinfo
 // ----------
 // Get server information
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StorageInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, false)
 
 	// Marshal API response
 	jsonBytes, err := json.Marshal(storageInfo)
@@ -290,31 +292,92 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 
 }
 
-// DataUsageInfoHandler - GET /minio/admin/v2/datausage
+// DataUsageInfoHandler - GET /minio/admin/v3/datausage
 // ----------
 // Get server/cluster data usage info
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DataUsageInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	dataUsageInfoJSON, err := json.Marshal(dataUsageInfo)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
+func (a adminAPIHandlers) AccountingUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AccountingUsageInfo")
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AccountingUsageInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var accountingUsageInfo = make(map[string]madmin.BucketAccountingUsage)
+
+	buckets, err := objectAPI.ListBuckets(ctx)
+	if err != nil {
+		// writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	users, err := globalIAMSys.ListUsers()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Load the latest calculated data usage
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+
+	// Calculate for each bucket, which users are allowed to access to it
+	for _, bucket := range buckets {
+		bucketUsageInfo := madmin.BucketAccountingUsage{}
+
+		// Fetch the data usage of the current bucket
+		if !dataUsageInfo.LastUpdate.IsZero() && dataUsageInfo.BucketsSizes != nil {
+			bucketUsageInfo.Size = dataUsageInfo.BucketsSizes[bucket.Name]
+		}
+
+		for user := range users {
+			rd, wr, custom := globalIAMSys.GetAccountAccess(user, bucket.Name)
+			if rd || wr || custom {
+				bucketUsageInfo.AccessList = append(bucketUsageInfo.AccessList, madmin.AccountAccess{
+					AccountName: user,
+					Read:        rd,
+					Write:       wr,
+					Custom:      custom,
+				})
+			}
+		}
+
+		accountingUsageInfo[bucket.Name] = bucketUsageInfo
+	}
+
+	usageInfoJSON, err := json.Marshal(accountingUsageInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, usageInfoJSON)
+}
+
 // ServerCPULoadInfo holds informantion about cpu utilization
 // of one minio node. It also reports any errors if encountered
 // while trying to reach this server.
@@ -342,14 +405,14 @@ type ServerNetReadPerfInfo struct {
 	Error          string `json:"error,omitempty"`
 }
 
-// PerfInfoHandler - GET /minio/admin/v2/performance?perfType={perfType}
+// PerfInfoHandler - GET /minio/admin/v3/performance?perfType={perfType}
 // ----------
 // Get all performance information based on input type
 // Supported types = drive
 func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PerfInfo")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.PerfInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -520,7 +583,7 @@ type PeerLocks struct {
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "TopLocks")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -559,42 +622,56 @@ type StartProfilingResult struct {
 	Error    string `json:"error"`
 }
 
-// StartProfilingHandler - POST /minio/admin/v2/profiling/start?profilerType={profilerType}
+// StartProfilingHandler - POST /minio/admin/v3/profiling/start?profilerType={profilerType}
 // ----------
 // Enable server profiling
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
 	vars := mux.Vars(r)
-	profiler := vars["profilerType"]
-
+	profiles := strings.Split(vars["profilerType"], ",")
 	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Start profiling on remote servers.
-	hostErrs := globalNotificationSys.StartProfiling(profiler)
+	globalProfilerMu.Lock()
+	defer globalProfilerMu.Unlock()
 
-	// Start profiling locally as well.
-	{
-		if globalProfiler != nil {
-			globalProfiler.Stop()
+	if globalProfiler == nil {
+		globalProfiler = make(map[string]minioProfiler, 10)
+	}
+
+	// Stop profiler of all types if already running
+	for k, v := range globalProfiler {
+		for _, p := range profiles {
+			if p == k {
+				v.Stop()
+				delete(globalProfiler, k)
+			}
 		}
-		prof, err := startProfiler(profiler, "")
+	}
+
+	// Start profiling on remote servers.
+	var hostErrs []NotificationPeerErr
+	for _, profiler := range profiles {
+		hostErrs = append(hostErrs, globalNotificationSys.StartProfiling(profiler)...)
+
+		// Start profiling locally as well.
+		prof, err := startProfiler(profiler)
 		if err != nil {
 			hostErrs = append(hostErrs, NotificationPeerErr{
 				Host: *thisAddr,
 				Err:  err,
 			})
 		} else {
-			globalProfiler = prof
+			globalProfiler[profiler] = prof
 			hostErrs = append(hostErrs, NotificationPeerErr{
 				Host: *thisAddr,
 			})
@@ -620,7 +697,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	writeSuccessResponseJSON(w, []byte(startProfilingResultInBytes))
+	writeSuccessResponseJSON(w, startProfilingResultInBytes)
 }
 
 // dummyFileInfo represents a dummy representation of a profile data file
@@ -641,13 +718,13 @@ func (f dummyFileInfo) ModTime() time.Time { return f.modTime }
 func (f dummyFileInfo) IsDir() bool        { return f.isDir }
 func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 
-// DownloadProfilingHandler - POST /minio/admin/v2/profiling/download
+// DownloadProfilingHandler - POST /minio/admin/v3/profiling/download
 // ----------
 // Download profiling information of all nodes in a zip format
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -713,7 +790,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 	if hip.clientToken == "" {
 		jerr := json.NewDecoder(r).Decode(&hip.hs)
 		if jerr != nil {
-			logger.LogIf(context.Background(), jerr, logger.Application)
+			logger.LogIf(GlobalContext, jerr, logger.Application)
 			err = ErrRequestBodyParse
 			return
 		}
@@ -723,7 +800,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 	return
 }
 
-// HealHandler - POST /minio/admin/v2/heal/
+// HealHandler - POST /minio/admin/v3/heal/
 // -----------
 // Start heal processing and return heal status items.
 //
@@ -821,7 +898,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// find number of disks in the setup
-	info := objectAPI.StorageInfo(ctx)
+	info := objectAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
 
 	healPath := pathJoin(hip.bucket, hip.objPrefix)
@@ -906,13 +983,24 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 
 	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{}
+	var aggregatedHealStateResult = madmin.BgHealState{
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
+		NextHealRound:     bgHealStates[0].NextHealRound,
+	}
+
+	bgHealStates = bgHealStates[1:]
+
 	for _, state := range bgHealStates {
 		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
 			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
 		}
-
 	}
 
 	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
@@ -979,6 +1067,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 
 	var apiErr APIError
 	switch e := err.(type) {
+	case iampolicy.Error:
+		apiErr = APIError{
+			Code:           "XMinioMalformedIAMPolicy",
+			Description:    e.Error(),
+			HTTPStatusCode: http.StatusBadRequest,
+		}
 	case config.Error:
 		apiErr = APIError{
 			Code:           "XMinioConfigError",
@@ -1022,7 +1116,7 @@ func mustTrace(entry interface{}, trcAll, errOnly bool) bool {
 	return trace
 }
 
-// TraceHandler - POST /minio/admin/v2/trace
+// TraceHandler - POST /minio/admin/v3/trace
 // ----------
 // The handler sends http trace to the connected HTTP client.
 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
@@ -1031,7 +1125,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	trcErr := r.URL.Query().Get("err") == "true"
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ListServerInfoAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.TraceAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -1085,7 +1179,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ConsoleLog")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1134,8 +1228,8 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	for {
 		select {
 		case entry := <-logCh:
-			log := entry.(madmin.LogInfo)
-			if log.SendLog(node, logKind) {
+			log, ok := entry.(log.Info)
+			if ok && log.SendLog(node, logKind) {
 				if err := enc.Encode(log); err != nil {
 					return
 				}
@@ -1152,11 +1246,11 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	}
 }
 
-// KMSKeyStatusHandler - GET /minio/admin/v2/kms/key/status?key-id=<master-key-id>
+// KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSKeyStatusHandler")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1188,20 +1282,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// 2. Check whether we can update / re-wrap the sealed key.
-	sealedKey, err = GlobalKMS.UpdateKey(keyID, sealedKey, kmsContext)
-	if err != nil {
-		response.UpdateErr = err.Error()
-		resp, err := json.Marshal(response)
-		if err != nil {
-			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
-			return
-		}
-		writeSuccessResponseJSON(w, resp)
-		return
-	}
-
-	// 3. Verify that we can indeed decrypt the (encrypted) key
+	// 2. Verify that we can indeed decrypt the (encrypted) key
 	decryptedKey, err := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
 	if err != nil {
 		response.DecryptionErr = err.Error()
@@ -1214,7 +1295,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// 4. Compare generated key with decrypted key
+	// 3. Compare generated key with decrypted key
 	if subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1 {
 		response.DecryptionErr = "The generated and the decrypted data key do not match"
 		resp, err := json.Marshal(response)
@@ -1234,14 +1315,14 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, resp)
 }
 
-// ServerHardwareInfoHandler - GET /minio/admin/v2/hardwareinfo?Type={hwType}
+// ServerHardwareInfoHandler - GET /minio/admin/v3/hardwareinfo?Type={hwType}
 // ----------
 // Get all hardware information based on input type
 // Supported types = cpu
 func (a adminAPIHandlers) ServerHardwareInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HardwareInfo")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerHardwareInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1291,12 +1372,141 @@ func (a adminAPIHandlers) ServerHardwareInfoHandler(w http.ResponseWriter, r *ht
 	}
 }
 
-// ServerInfoHandler - GET /minio/admin/v2/info
+// OBDInfoHandler - GET /minio/admin/v3/obdinfo
+// ----------
+// Get server on-board diagnostics
+func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "OBDInfo")
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.OBDInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+	deadlinedCtx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Minute))
+	obdInfo := madmin.OBDInfo{}
+
+	setCommonHeaders(w)
+	w.Header().Set(xhttp.ContentType, string(mimeJSON))
+	w.WriteHeader(http.StatusOK)
+
+	enc := json.NewEncoder(w)
+	partialWrite := func() {
+		logger.LogIf(ctx, enc.Encode(obdInfo))
+	}
+
+	finish := func() {
+		partialWrite()
+		w.(http.Flusher).Flush()
+		cancel()
+	}
+
+	errResp := func(err error) {
+		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
+			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+		encodedErrorResponse := encodeResponse(errorResponse)
+		obdInfo.Error = string(encodedErrorResponse)
+		finish()
+	}
+
+	nsLock := objectAPI.NewNSLock(deadlinedCtx, minioMetaBucket, "obd-in-progress")
+	if err := nsLock.GetLock(newDynamicTimeout(10*time.Minute, 600*time.Second)); err != nil { // returns a locked lock
+		errResp(err)
+		return
+	}
+	defer nsLock.Unlock()
+
+	vars := mux.Vars(r)
+
+	if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
+		cpuInfo := getLocalCPUOBDInfo(deadlinedCtx)
+
+		obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
+		obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, globalNotificationSys.CPUOBDInfo(deadlinedCtx)...)
+		partialWrite()
+	}
+
+	if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" {
+		diskHwInfo := getLocalDiskHwOBD(deadlinedCtx)
+
+		obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
+		obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwOBDInfo(deadlinedCtx)...)
+		partialWrite()
+	}
+
+	if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" {
+		osInfo := getLocalOsInfoOBD(deadlinedCtx)
+
+		obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
+		obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, globalNotificationSys.OsOBDInfo(deadlinedCtx)...)
+		partialWrite()
+	}
+
+	if mem, ok := vars["sysmem"]; ok && mem == "true" {
+		memInfo := getLocalMemOBD(deadlinedCtx)
+
+		obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
+		obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, globalNotificationSys.MemOBDInfo(deadlinedCtx)...)
+		partialWrite()
+	}
+
+	if proc, ok := vars["sysprocess"]; ok && proc == "true" {
+		procInfo := getLocalProcOBD(deadlinedCtx)
+
+		obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
+		obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, globalNotificationSys.ProcOBDInfo(deadlinedCtx)...)
+		partialWrite()
+	}
+
+	if config, ok := vars["minioconfig"]; ok && config == "true" {
+		cfg, err := readServerConfig(ctx, objectAPI)
+		logger.LogIf(ctx, err)
+		obdInfo.Minio.Config = cfg
+		partialWrite()
+	}
+
+	if drive, ok := vars["perfdrive"]; ok && drive == "true" {
+		// Get drive obd details from local server's drive(s)
+		driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
+		driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
+
+		errStr := ""
+		if driveOBDSerial.Error != "" {
+			errStr = "serial: " + driveOBDSerial.Error
+		}
+		if driveOBDParallel.Error != "" {
+			errStr = errStr + " parallel: " + driveOBDParallel.Error
+		}
+
+		driveOBD := madmin.ServerDrivesOBDInfo{
+			Addr:     driveOBDSerial.Addr,
+			Serial:   driveOBDSerial.Serial,
+			Parallel: driveOBDParallel.Parallel,
+			Error:    errStr,
+		}
+		obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBD)
+
+		// Notify all other MinIO peers to report drive obd numbers
+		driveOBDs := globalNotificationSys.DriveOBDInfo(deadlinedCtx)
+		obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBDs...)
+
+		partialWrite()
+	}
+
+	if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistXL {
+		obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
+		obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.DispatchNetOBDInfo(deadlinedCtx)...)
+		obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
+		partialWrite()
+	}
+
+	finish()
+}
+
+// ServerInfoHandler - GET /minio/admin/v3/info
 // ----------
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, "")
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1329,10 +1539,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		} else if ldapConn == nil {
 			ldap.Status = "Not Configured"
 		} else {
+			// Close ldap connection to avoid leaks.
+			ldapConn.Close()
 			ldap.Status = "online"
 		}
-		// Close ldap connection to avoid leaks.
-		defer ldapConn.Close()
 	}
 
 	log, audit := fetchLoggerInfo(cfg)
@@ -1341,7 +1551,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	notifyTarget := fetchLambdaInfo(cfg)
 
 	// Fetching the Storage information
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, false)
 
 	var OnDisks int
 	var OffDisks int
@@ -1356,7 +1566,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 			OffDisks += v
 		}
 
-		backend = madmin.XlBackend{
+		backend = madmin.XLBackend{
 			Type:             madmin.ErasureType,
 			OnlineDisks:      OnDisks,
 			OfflineDisks:     OffDisks,
@@ -1366,7 +1576,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 			RRSCParity:       storageInfo.Backend.RRSCParity,
 		}
 	} else {
-		backend = madmin.FsBackend{
+		backend = madmin.FSBackend{
 			Type: madmin.FsType,
 		}
 	}
@@ -1450,8 +1660,13 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 }
 
 func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
+
+	// Fetch the targets
+	targetList, err := notify.RegisterNotificationTargets(cfg, GlobalServiceDoneCh, NewGatewayHTTPTransport(), nil, true)
+	if err != nil {
+		return nil
+	}
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
-	targetList, _ := notify.GetNotificationTargets(cfg, GlobalServiceDoneCh, NewCustomHTTPTransport())
 
 	for targetID, target := range targetList.TargetMap() {
 		targetIDStatus := make(map[string]madmin.Status)
@@ -1464,6 +1679,8 @@ func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 		list := lambdaMap[targetID.Name]
 		list = append(list, targetIDStatus)
 		lambdaMap[targetID.Name] = list
+		// Close any leaking connections
+		_ = target.Close()
 	}
 
 	notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap))
@@ -1492,36 +1709,29 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 		return vault
 	}
 
-	if err := checkConnection(kmsInfo.Endpoint); err != nil {
+	if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil {
 
 		vault.Status = "offline"
 	} else {
 		vault.Status = "online"
 
-		kmsContext := crypto.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation
+		kmsContext := crypto.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
 		// 1. Generate a new key using the KMS.
 		key, sealedKey, err := GlobalKMS.GenerateKey(keyID, kmsContext)
 		if err != nil {
-			vault.Encrypt = "Encryption failed"
+			vault.Encrypt = fmt.Sprintf("Encryption failed: %v", err)
 		} else {
 			vault.Encrypt = "Ok"
 		}
 
-		// 2. Check whether we can update / re-wrap the sealed key.
-		sealedKey, err = GlobalKMS.UpdateKey(keyID, sealedKey, kmsContext)
-		if err != nil {
-			vault.Update = "Re-wrap failed:"
-		} else {
-			vault.Update = "Ok"
-		}
-
-		// 3. Verify that we can indeed decrypt the (encrypted) key
-		decryptedKey, decryptErr := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
-
-		// 4. Compare generated key with decrypted key
-		if subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1 || decryptErr != nil {
-			vault.Decrypt = "Re-wrap failed:"
-		} else {
+		// 2. Verify that we can indeed decrypt the (encrypted) key
+		decryptedKey, err := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
+		switch {
+		case err != nil:
+			vault.Decrypt = fmt.Sprintf("Decryption failed: %v", err)
+		case subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1:
+			vault.Decrypt = "Decryption failed: decrypted key does not match generated key"
+		default:
 			vault.Decrypt = "Ok"
 		}
 	}
@@ -1536,7 +1746,7 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 	var auditlogger []madmin.Audit
 	for log, l := range loggerCfg.HTTP {
 		if l.Enabled {
-			err := checkConnection(l.Endpoint)
+			err := checkConnection(l.Endpoint, 15*time.Second)
 			if err == nil {
 				mapLog := make(map[string]madmin.Status)
 				mapLog[log] = madmin.Status{Status: "Online"}
@@ -1551,7 +1761,7 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 
 	for audit, l := range loggerCfg.Audit {
 		if l.Enabled {
-			err := checkConnection(l.Endpoint)
+			err := checkConnection(l.Endpoint, 15*time.Second)
 			if err == nil {
 				mapAudit := make(map[string]madmin.Status)
 				mapAudit[audit] = madmin.Status{Status: "Online"}
@@ -1567,12 +1777,19 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 }
 
 // checkConnection - ping an endpoint , return err in case of no connection
-func checkConnection(endpointStr string) error {
+func checkConnection(endpointStr string, timeout time.Duration) error {
 	u, pErr := xnet.ParseURL(endpointStr)
 	if pErr != nil {
 		return pErr
 	}
-	if dErr := u.DialHTTP(); dErr != nil {
+
+	tr := newCustomHTTPTransport(
+		&tls.Config{RootCAs: globalRootCAs},
+		timeout,
+		0, /* Default value */
+	)()
+
+	if dErr := u.DialHTTP(tr); dErr != nil {
 		if urlErr, ok := dErr.(*url.Error); ok {
 			// To treat "connection refused" errors as un reachable endpoint.
 			if target.IsConnRefusedErr(urlErr.Err) {

cmd/admin-handlers_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
 	"io"
 	"io/ioutil"
@@ -47,6 +46,10 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// reset global variables to start afresh.
 	resetTestGlobals()
 
+	// Set globalIsXL to indicate that the setup uses an erasure
+	// code backend.
+	globalIsXL = true
+
 	// Initializing objectLayer for HealFormatHandler.
 	objLayer, xlDirs, xlErr := initTestXLObjLayer()
 	if xlErr != nil {
@@ -63,21 +66,12 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 
 	globalEndpoints = mustGetZoneEndpoints(xlDirs...)
 
-	// Set globalIsXL to indicate that the setup uses an erasure
-	// code backend.
-	globalIsXL = true
-
-	// Init global heal state
-	if globalIsXL {
-		globalAllHealState = initHealState()
-	}
-
 	globalConfigSys = NewConfigSys()
 
 	globalIAMSys = NewIAMSys()
-	globalIAMSys.Init(objLayer)
+	globalIAMSys.Init(GlobalContext, objLayer)
 
-	buckets, err := objLayer.ListBuckets(context.Background())
+	buckets, err := objLayer.ListBuckets(GlobalContext)
 	if err != nil {
 		return nil, err
 	}
@@ -114,14 +108,14 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) {
 		return nil, nil, err
 	}
 	endpoints := mustGetNewEndpoints(xlDirs...)
-	format, err := waitForFormatXL(true, endpoints, 1, 16, "")
+	storageDisks, format, err := waitForFormatXL(true, endpoints, 1, 1, 16, "")
 	if err != nil {
 		removeRoots(xlDirs)
 		return nil, nil, err
 	}
 
 	globalPolicySys = NewPolicySys()
-	objLayer, err := newXLSets(endpoints, format, 1, 16)
+	objLayer, err := newXLSets(endpoints, storageDisks, format, 1, 16)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -268,6 +262,7 @@ func TestAdminServerInfo(t *testing.T) {
 	if err != nil {
 		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
+
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
@@ -318,12 +313,12 @@ func TestToAdminAPIErrCode(t *testing.T) {
 		// 3. Non-admin API specific error.
 		{
 			err:            errDiskNotFound,
-			expectedAPIErr: toAPIErrorCode(context.Background(), errDiskNotFound),
+			expectedAPIErr: toAPIErrorCode(GlobalContext, errDiskNotFound),
 		},
 	}
 
 	for i, test := range testCases {
-		actualErr := toAdminAPIErrCode(context.Background(), test.err)
+		actualErr := toAdminAPIErrCode(GlobalContext, test.err)
 		if actualErr != test.expectedAPIErr {
 			t.Errorf("Test %d: Expected %v but received %v",
 				i+1, test.expectedAPIErr, actualErr)

cmd/admin-heal-ops.go

@@ -106,12 +106,12 @@ func initHealState() *allHealState {
 		healSeqMap: make(map[string]*healSequence),
 	}
 
-	go healState.periodicHealSeqsClean()
+	go healState.periodicHealSeqsClean(GlobalContext)
 
 	return healState
 }
 
-func (ahs *allHealState) periodicHealSeqsClean() {
+func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
 	ticker := time.NewTicker(time.Minute * 5)
@@ -127,7 +127,7 @@ func (ahs *allHealState) periodicHealSeqsClean() {
 				}
 			}
 			ahs.Unlock()
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			// server could be restarting - need
 			// to exit immediately
 			return
@@ -183,7 +183,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 	}
 
 	b, err := json.Marshal(&hsp)
-	return b, toAdminAPIErr(context.Background(), err)
+	return b, toAdminAPIErr(GlobalContext, err)
 }
 
 // LaunchNewHealSequence - launches a background routine that performs
@@ -306,6 +306,12 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 	return jbytes, ErrNone
 }
 
+// healSource denotes single entity and heal option.
+type healSource struct {
+	path string           // entity path (format, buckets, objects) to heal
+	opts *madmin.HealOpts // optional heal option overrides default setting
+}
+
 // healSequence - state for each heal sequence initiated on the
 // server.
 type healSequence struct {
@@ -316,11 +322,9 @@ type healSequence struct {
 	path string
 
 	// List of entities (format, buckets, objects) to heal
-	sourceCh chan string
+	sourceCh chan healSource
 
-	// Report healing progress, false if this is a background
-	// healing since currently there is no entity which will
-	// receive realtime healing status
+	// Report healing progress
 	reportProgress bool
 
 	// time at which heal sequence was started
@@ -352,14 +356,23 @@ type healSequence struct {
 	// the last result index sent to client
 	lastSentResultIndex int64
 
-	// Number of total items scanned
-	scannedItemsCount int64
+	// Number of total items scanned against item type
+	scannedItemsMap map[madmin.HealItemType]int64
+
+	// Number of total items healed against item type
+	healedItemsMap map[madmin.HealItemType]int64
+
+	// Number of total items where healing failed against endpoint and drive state
+	healFailedItemsMap map[string]int64
 
 	// The time of the last scan/heal activity
 	lastHealActivity time.Time
 
 	// Holds the request-info for logging
 	ctx context.Context
+
+	// used to lock this structure as it is concurrently accessed
+	mutex sync.RWMutex
 }
 
 // NewHealSequence - creates healSettings, assumes bucket and
@@ -369,7 +382,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 
 	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
 	reqInfo.AppendTags("prefix", objPrefix)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 
 	return &healSequence{
 		bucket:         bucket,
@@ -390,9 +403,83 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 		traverseAndHealDoneCh: make(chan error),
 		stopSignalCh:          make(chan struct{}),
 		ctx:                   ctx,
+		scannedItemsMap:       make(map[madmin.HealItemType]int64),
+		healedItemsMap:        make(map[madmin.HealItemType]int64),
+		healFailedItemsMap:    make(map[string]int64),
 	}
 }
 
+// resetHealStatusCounters - reset the healSequence status counters between
+// each monthly background heal scanning activity.
+// This is used only in case of Background healing scenario, where
+// we use a single long running healSequence which reactively heals
+// objects passed to the SourceCh.
+func (h *healSequence) resetHealStatusCounters() {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+
+	h.currentStatus.Items = []madmin.HealResultItem{}
+	h.lastSentResultIndex = 0
+	h.scannedItemsMap = make(map[madmin.HealItemType]int64)
+	h.healedItemsMap = make(map[madmin.HealItemType]int64)
+	h.healFailedItemsMap = make(map[string]int64)
+}
+
+// getScannedItemsCount - returns a count of all scanned items
+func (h *healSequence) getScannedItemsCount() int64 {
+	var count int64
+	h.mutex.RLock()
+	defer h.mutex.RUnlock()
+
+	for _, v := range h.scannedItemsMap {
+		count = count + v
+	}
+	return count
+}
+
+// getScannedItemsMap - returns map of all scanned items against type
+func (h *healSequence) getScannedItemsMap() map[madmin.HealItemType]int64 {
+	h.mutex.RLock()
+	defer h.mutex.RUnlock()
+
+	// Make a copy before returning the value
+	retMap := make(map[madmin.HealItemType]int64, len(h.scannedItemsMap))
+	for k, v := range h.scannedItemsMap {
+		retMap[k] = v
+	}
+
+	return retMap
+}
+
+// getHealedItemsMap - returns the map of all healed items against type
+func (h *healSequence) getHealedItemsMap() map[madmin.HealItemType]int64 {
+	h.mutex.RLock()
+	defer h.mutex.RUnlock()
+
+	// Make a copy before returning the value
+	retMap := make(map[madmin.HealItemType]int64, len(h.healedItemsMap))
+	for k, v := range h.healedItemsMap {
+		retMap[k] = v
+	}
+
+	return retMap
+}
+
+// gethealFailedItemsMap - returns map of all items where heal failed against
+// drive endpoint and status
+func (h *healSequence) gethealFailedItemsMap() map[string]int64 {
+	h.mutex.RLock()
+	defer h.mutex.RUnlock()
+
+	// Make a copy before returning the value
+	retMap := make(map[string]int64, len(h.healFailedItemsMap))
+	for k, v := range h.healFailedItemsMap {
+		retMap[k] = v
+	}
+
+	return retMap
+}
+
 // isQuitting - determines if the heal sequence is quitting (due to an
 // external signal)
 func (h *healSequence) isQuitting() bool {
@@ -548,14 +635,38 @@ func (h *healSequence) healSequenceStart() {
 	}
 }
 
-func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType) error {
+func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
 	var respCh = make(chan healResult)
 	defer close(respCh)
 	// Send heal request
-	globalBackgroundHealRoutine.queueHealTask(healTask{path: path, responseCh: respCh, opts: h.settings})
+	task := healTask{
+		path:       source.path,
+		responseCh: respCh,
+		opts:       h.settings,
+	}
+	if source.opts != nil {
+		task.opts = *source.opts
+	}
+	globalBackgroundHealRoutine.queueHealTask(task)
 	// Wait for answer and push result to the client
 	res := <-respCh
 	if !h.reportProgress {
+		h.mutex.Lock()
+		defer h.mutex.Unlock()
+
+		// Progress is not reported in case of background heal processing.
+		// Instead we increment relevant counter based on the heal result
+		// for prometheus reporting.
+		if res.err != nil && !isErrObjectNotFound(res.err) {
+			for _, d := range res.result.After.Drives {
+				// For failed items we report the endpoint and drive state
+				// This will help users take corrective actions for drives
+				h.healFailedItemsMap[d.Endpoint+","+d.State]++
+			}
+		} else {
+			// Only object type reported for successful healing
+			h.healedItemsMap[res.result.Type]++
+		}
 		return nil
 	}
 	res.result.Type = healType
@@ -575,46 +686,38 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
-	h.lastHealActivity = UTCNow()
-
-	// Start healing the config prefix.
-	if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
+	bucketsOnly := true // heal buckets only, not objects.
+	if err := h.healItems(bucketsOnly); err != nil {
 		logger.LogIf(h.ctx, err)
 	}
 
-	// Start healing the bucket config prefix.
-	if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
+	for {
+		select {
+		case source := <-h.sourceCh:
+			var itemType madmin.HealItemType
+			switch {
+			case source.path == nopHeal:
+				continue
+			case source.path == SlashSeparator:
+				itemType = madmin.HealItemMetadata
+			case !strings.Contains(source.path, SlashSeparator):
+				itemType = madmin.HealItemBucket
+			default:
+				itemType = madmin.HealItemObject
+			}
 
-	// Start healing the background ops prefix.
-	if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
+			if err := h.queueHealTask(source, itemType); err != nil {
+				logger.LogIf(h.ctx, err)
+			}
 
-	for path := range h.sourceCh {
-
-		var itemType madmin.HealItemType
-		switch {
-		case path == nopHeal:
-			continue
-		case path == SlashSeparator:
-			itemType = madmin.HealItemMetadata
-		case !strings.Contains(path, SlashSeparator):
-			itemType = madmin.HealItemBucket
-		default:
-			itemType = madmin.HealItemObject
+			h.scannedItemsMap[itemType]++
+			h.lastHealActivity = UTCNow()
+		case <-h.traverseAndHealDoneCh:
+			return nil
+		case <-h.ctx.Done():
+			return nil
 		}
-
-		if err := h.queueHealTask(path, itemType); err != nil {
-			logger.LogIf(h.ctx, err)
-		}
-
-		h.scannedItemsCount++
-		h.lastHealActivity = UTCNow()
 	}
-
-	return nil
 }
 
 func (h *healSequence) healFromSourceCh() {
@@ -622,7 +725,7 @@ func (h *healSequence) healFromSourceCh() {
 	close(h.traverseAndHealDoneCh)
 }
 
-func (h *healSequence) healItems() error {
+func (h *healSequence) healItems(bucketsOnly bool) error {
 	// Start with format healing
 	if err := h.healDiskFormat(); err != nil {
 		return err
@@ -638,13 +741,8 @@ func (h *healSequence) healItems() error {
 		return err
 	}
 
-	// Start healing the background ops prefix.
-	if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
-
 	// Heal buckets and objects
-	return h.healBuckets()
+	return h.healBuckets(bucketsOnly)
 }
 
 // traverseAndHeal - traverses on-disk data and performs healing
@@ -655,7 +753,8 @@ func (h *healSequence) healItems() error {
 // has to wait until a safe point is reached, such as between scanning
 // two objects.
 func (h *healSequence) traverseAndHeal() {
-	if err := h.healItems(); err != nil {
+	bucketsOnly := false // Heals buckets and objects also.
+	if err := h.healItems(bucketsOnly); err != nil {
 		if h.isQuitting() {
 			err = errHealStopSignalled
 		}
@@ -678,12 +777,12 @@ func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, func(bucket string, object string) error {
+		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket string, object string) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
 
-			herr := h.queueHealTask(pathJoin(bucket, object), madmin.HealItemBucketMetadata)
+			herr := h.queueHealTask(healSource{path: pathJoin(bucket, object)}, madmin.HealItemBucketMetadata)
 			// Object might have been deleted, by the time heal
 			// was attempted we ignore this object an move on.
 			if isErrObjectNotFound(herr) {
@@ -707,18 +806,18 @@ func (h *healSequence) healDiskFormat() error {
 		return errServerNotInitialized
 	}
 
-	return h.queueHealTask(SlashSeparator, madmin.HealItemMetadata)
+	return h.queueHealTask(healSource{path: SlashSeparator}, madmin.HealItemMetadata)
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
-func (h *healSequence) healBuckets() error {
+func (h *healSequence) healBuckets(bucketsOnly bool) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
 
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
-		return h.healBucket(h.bucket)
+		return h.healBucket(h.bucket, bucketsOnly)
 	}
 
 	// Get current object layer instance.
@@ -733,7 +832,7 @@ func (h *healSequence) healBuckets() error {
 	}
 
 	for _, bucket := range buckets {
-		if err = h.healBucket(bucket.Name); err != nil {
+		if err = h.healBucket(bucket.Name, bucketsOnly); err != nil {
 			return err
 		}
 	}
@@ -742,17 +841,21 @@ func (h *healSequence) healBuckets() error {
 }
 
 // healBucket - traverses and heals given bucket
-func (h *healSequence) healBucket(bucket string) error {
+func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
-	if err := h.queueHealTask(bucket, madmin.HealItemBucket); err != nil {
+	if err := h.queueHealTask(healSource{path: bucket}, madmin.HealItemBucket); err != nil {
 		return err
 	}
 
+	if bucketsOnly {
+		return nil
+	}
+
 	if !h.settings.Recursive {
 		if h.objPrefix != "" {
 			// Check if an object named as the objPrefix exists,
@@ -768,7 +871,7 @@ func (h *healSequence) healBucket(bucket string) error {
 		return nil
 	}
 
-	if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.healObject); err != nil {
+	if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.settings, h.healObject); err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
 	return nil
@@ -776,15 +879,15 @@ func (h *healSequence) healBucket(bucket string) error {
 
 // healObject - heal the given object and record result
 func (h *healSequence) healObject(bucket, object string) error {
-	if h.isQuitting() {
-		return errHealStopSignalled
-	}
-
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
-	return h.queueHealTask(pathJoin(bucket, object), madmin.HealItemObject)
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
+	return h.queueHealTask(healSource{path: pathJoin(bucket, object)}, madmin.HealItemObject)
 }

cmd/admin-router.go

@@ -24,9 +24,11 @@ import (
 )
 
 const (
-	adminPathPrefix       = minioReservedBucketPath + "/admin"
-	adminAPIVersion       = madmin.AdminAPIVersion
-	adminAPIVersionPrefix = SlashSeparator + madmin.AdminAPIVersion
+	adminPathPrefix         = minioReservedBucketPath + "/admin"
+	adminAPIVersionV2       = madmin.AdminAPIVersionV2
+	adminAPIVersion         = madmin.AdminAPIVersion
+	adminAPIVersionPrefix   = SlashSeparator + adminAPIVersion
+	adminAPIVersionV2Prefix = SlashSeparator + adminAPIVersionV2
 )
 
 // adminAPIHandlers provides HTTP handlers for MinIO admin API.
@@ -41,125 +43,152 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 	/// Service operations
 
-	// Restart and stop MinIO service.
-	adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix+"/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
-	// Update MinIO servers.
-	adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix+"/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
-
-	// Info operations
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
-	// Harware Info operations
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/hardware").HandlerFunc(httpTraceAll(adminAPI.ServerHardwareInfoHandler)).Queries("hwType", "{hwType:.*}")
-
-	// StorageInfo operations
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/storageinfo").HandlerFunc(httpTraceAll(adminAPI.StorageInfoHandler))
-	// DataUsageInfo operations
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
-
-	if globalIsDistXL || globalIsXL {
-		/// Heal operations
-
-		// Heal processing endpoint.
-		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-
-		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
-
-		/// Health operations
-
-	}
-	// Performance command - return performance details based on input type
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
-
-	// Profiling operations
-	adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix+"/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
-		Queries("profilerType", "{profilerType:.*}")
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
-
-	// Config KV operations.
-	if enableConfigOps {
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/get-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKVHandler)).Queries("key", "{key:.*}")
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/set-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKVHandler))
-		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix + "/del-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.DelConfigKVHandler))
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)).Queries("count", "{count:[0-9]+}")
-		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/clear-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/restore-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
+	adminVersions := []string{
+		adminAPIVersionPrefix,
+		adminAPIVersionV2Prefix,
 	}
 
-	/// Config operations
-	if enableConfigOps {
-		// Get config
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
-		// Set config
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
+	for _, adminVersion := range adminVersions {
+		// Restart and stop MinIO service.
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
+		// Update MinIO servers.
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
+
+		// Info operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
+		// Harware Info operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/hardware").HandlerFunc(httpTraceAll(adminAPI.ServerHardwareInfoHandler)).Queries("hwType", "{hwType:.*}")
+
+		// StorageInfo operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(httpTraceAll(adminAPI.StorageInfoHandler))
+		// DataUsageInfo operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
+
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountingusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountingUsageInfoHandler))
+
+		if globalIsDistXL || globalIsXL {
+			/// Heal operations
+
+			// Heal processing endpoint.
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
+
+			/// Health operations
+
+		}
+		// Performance command - return performance details based on input type
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
+
+		// Profiling operations
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
+			Queries("profilerType", "{profilerType:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
+
+		// Config KV operations.
+		if enableConfigOps {
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKVHandler)).Queries("key", "{key:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/set-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKVHandler))
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/del-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.DelConfigKVHandler))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)).Queries("count", "{count:[0-9]+}")
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/clear-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
+		}
+
+		/// Config operations
+		if enableConfigOps {
+			// Get config
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
+			// Set config
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
+		}
+
+		if enableIAMOps {
+			// -- IAM APIs --
+
+			// Add policy IAM
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
+
+			// Add user IAM
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
+
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
+
+			// Service accounts ops
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(httpTraceHdrs(adminAPI.AddServiceAccount))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-service-account").HandlerFunc(httpTraceHdrs(adminAPI.GetServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+
+			if adminVersion == adminAPIVersionV2Prefix {
+				// Info policy IAM v2
+				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicyV2)).Queries("name", "{name:.*}")
+
+				// List policies v2
+				adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPoliciesV2))
+			} else {
+				// Info policy IAM latest
+				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
+
+				// List policies latest
+				adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+			}
+
+			// Remove policy IAM
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+
+			// Set user or group policy
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-or-group-policy").
+				HandlerFunc(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup)).
+				Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
+
+			// Remove user IAM
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+
+			// List users
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+
+			// User info
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
+
+			// Add/Remove members from group
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
+
+			// Get Group
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
+
+			// List Groups
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
+
+			// Set Group Status
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
+
+		}
+
+		// -- Top APIs --
+		// Top locks
+		if globalIsDistXL {
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+		}
+
+		// HTTP Trace
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(adminAPI.TraceHandler)
+
+		// Console Logs
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/log").HandlerFunc(httpTraceAll(adminAPI.ConsoleLogHandler))
+
+		// -- KMS APIs --
+		//
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
+
+		if !globalIsGateway {
+
+			// -- OBD API --
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).Queries("perfdrive", "{perfdrive:true|false}", "perfnet", "{perfnet:true|false}", "minioinfo", "{minioinfo:true|false}", "minioconfig", "{minioconfig:true|false}", "syscpu", "{syscpu:true|false}", "sysdiskhw", "{sysdiskhw:true|false}", "sysosinfo", "{sysosinfo:true|false}", "sysmem", "{sysmem:true|false}", "sysprocess", "{sysprocess:true|false}")
+		}
 	}
 
-	if enableIAMOps {
-		// -- IAM APIs --
-
-		// Add policy IAM
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name",
-			"{name:.*}")
-
-		// Add user IAM
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
-			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
-
-		// Info policy IAM
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
-
-		// Remove policy IAM
-		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
-
-		// Set user or group policy
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-user-or-group-policy").
-			HandlerFunc(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup)).
-			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
-
-		// Remove user IAM
-		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
-
-		// List users
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
-
-		// User info
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
-
-		// Add/Remove members from group
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
-
-		// Get Group
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
-
-		// List Groups
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
-
-		// Set Group Status
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
-
-		// List policies
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
-	}
-
-	// -- Top APIs --
-	// Top locks
-	if globalIsDistXL {
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
-	}
-
-	// HTTP Trace
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/trace").HandlerFunc(adminAPI.TraceHandler)
-
-	// Console Logs
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/log").HandlerFunc(httpTraceAll(adminAPI.ConsoleLogHandler))
-
-	// -- KMS APIs --
-	//
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
-
 	// If none of the routes match add default error handler routes
 	adminRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))
 	adminRouter.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))

cmd/admin-server-info.go

@@ -20,9 +20,8 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"strings"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/madmin"
@@ -190,51 +189,49 @@ func getLocalNetworkInfo(endpointZones EndpointZones, r *http.Request) madmin.Se
 // getLocalServerProperty - returns ServerDrivesPerfInfo for only the
 // local endpoints from given list of endpoints
 func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
-	var di madmin.Disk
 	var disks []madmin.Disk
 	addr := r.Host
 	if globalIsDistXL {
 		addr = GetLocalPeer(endpointZones)
 	}
 	network := make(map[string]string)
-	hosts := set.NewStringSet()
 	for _, ep := range endpointZones {
 		for _, endpoint := range ep.Endpoints {
-
-			url := strings.Replace(endpoint.URL.String(), endpoint.Path, "", -1)
-			if url == "" {
-				url = r.Host
+			nodeName := endpoint.Host
+			if nodeName == "" {
+				nodeName = r.Host
 			}
-			hosts.Add(url)
-
-			// Only proceed for local endpoints
 			if endpoint.IsLocal {
-				url = fetchAddress(url)
-				network[url] = "online"
-				if _, err := os.Stat(endpoint.Path); err != nil {
-					continue
+				// Only proceed for local endpoints
+				network[nodeName] = "online"
+				var di = madmin.Disk{
+					DrivePath: endpoint.Path,
+				}
+				diInfo, err := disk.GetInfo(endpoint.Path)
+				if err != nil {
+					if os.IsNotExist(err) || isSysErrPathNotFound(err) {
+						di.State = madmin.DriveStateMissing
+					} else {
+						di.State = madmin.DriveStateCorrupt
+					}
+				} else {
+					di.State = madmin.DriveStateOk
+					di.DrivePath = endpoint.Path
+					di.TotalSpace = diInfo.Total
+					di.UsedSpace = diInfo.Total - diInfo.Free
+					di.Utilization = float64((diInfo.Total - diInfo.Free) / diInfo.Total * 100)
 				}
-
-				diInfo, _ := disk.GetInfo(endpoint.Path)
-				di.State = "ok"
-				di.DrivePath = endpoint.Path
-				di.TotalSpace = diInfo.Total
-				di.UsedSpace = diInfo.Total - diInfo.Free
-				di.Utilization = float64((diInfo.Total - diInfo.Free) / diInfo.Total * 100)
 				disks = append(disks, di)
-			}
-		}
-	}
-
-	for host := range hosts {
-		_, present := network[host]
-		if !present {
-			err := checkConnection(host)
-			host = fetchAddress(host)
-			if err != nil {
-				network[host] = "offline"
 			} else {
-				network[host] = "online"
+				_, present := network[nodeName]
+				if !present {
+					err := IsServerResolvable(endpoint)
+					if err == nil {
+						network[nodeName] = "online"
+					} else {
+						network[nodeName] = "offline"
+					}
+				}
 			}
 		}
 	}
@@ -249,13 +246,3 @@ func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin
 		Disks:    disks,
 	}
 }
-
-// Replaces http and https from address
-func fetchAddress(address string) string {
-	if strings.Contains(address, "http://") {
-		address = strings.Replace(address, "http://", "", -1)
-	} else if strings.Contains(address, "https://") {
-		address = strings.Replace(address, "https://", "", -1)
-	}
-	return address
-}

cmd/api-errors.go

@@ -21,6 +21,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
@@ -32,6 +33,10 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/object/tagging"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 )
@@ -90,9 +95,12 @@ const (
 	ErrMissingContentLength
 	ErrMissingContentMD5
 	ErrMissingRequestBodyError
+	ErrMissingSecurityHeader
 	ErrNoSuchBucket
 	ErrNoSuchBucketPolicy
 	ErrNoSuchBucketLifecycle
+	ErrNoSuchLifecycleConfiguration
+	ErrNoSuchBucketSSEConfig
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrNoSuchVersion
@@ -114,7 +122,8 @@ const (
 	ErrMissingCredTag
 	ErrCredMalformed
 	ErrInvalidRegion
-	ErrInvalidService
+	ErrInvalidServiceS3
+	ErrInvalidServiceSTS
 	ErrInvalidRequestVersion
 	ErrMissingSignTag
 	ErrMissingSignHeadersTag
@@ -142,11 +151,14 @@ const (
 	ErrBadRequest
 	ErrKeyTooLongError
 	ErrInvalidBucketObjectLockConfiguration
+	ErrObjectLockConfigurationNotAllowed
+	ErrNoSuchObjectLockConfiguration
 	ErrObjectLocked
 	ErrInvalidRetentionDate
 	ErrPastObjectLockRetainDate
 	ErrUnknownWORMModeDirective
 	ErrObjectLockInvalidHeaders
+	ErrInvalidTagDirective
 	// Add new error codes here.
 
 	// SSE-S3 related API errors
@@ -199,6 +211,7 @@ const (
 	ErrInvalidResourceName
 	ErrServerNotInitialized
 	ErrOperationTimedOut
+	ErrOperationMaxedOut
 	ErrInvalidRequest
 	// MinIO storage class error codes
 	ErrInvalidStorageClass
@@ -321,6 +334,7 @@ const (
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
+	ErrAddServiceAccountInvalidArgument
 	ErrPostPolicyConditionInvalidFormat
 )
 
@@ -462,6 +476,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Missing required header for this request: Content-Md5.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMissingSecurityHeader: {
+		Code:           "MissingSecurityHeader",
+		Description:    "Your request was missing a required header",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrMissingRequestBodyError: {
 		Code:           "MissingRequestBodyError",
 		Description:    "Request body is empty.",
@@ -482,6 +501,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The bucket lifecycle configuration does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrNoSuchLifecycleConfiguration: {
+		Code:           "NoSuchLifecycleConfiguration",
+		Description:    "The lifecycle configuration does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchBucketSSEConfig: {
+		Code:           "ServerSideEncryptionConfigurationNotFoundError",
+		Description:    "The server side encryption configuration was not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNoSuchKey: {
 		Code:           "NoSuchKey",
 		Description:    "The specified key does not exist.",
@@ -626,9 +655,14 @@ var errorCodes = errorCodeMap{
 	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
 	// right Description:   "Error parsing the X-Amz-Credential parameter; incorrect service \"s4\". This endpoint belongs to \"s3\".".
 	// Need changes to make sure variable messages can be constructed.
-	ErrInvalidService: {
-		Code:           "AuthorizationQueryParametersError",
-		Description:    "Error parsing the X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
+	ErrInvalidServiceS3: {
+		Code:           "AuthorizationParametersError",
+		Description:    "Error parsing the Credential/X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidServiceSTS: {
+		Code:           "AuthorizationParametersError",
+		Description:    "Error parsing the Credential parameter; incorrect service. This endpoint belongs to \"sts\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
@@ -731,6 +765,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Bucket is missing ObjectLockConfiguration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectLockConfigurationNotAllowed: {
+		Code:           "InvalidBucketState",
+		Description:    "Object Lock configuration cannot be enabled on existing buckets.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrNoSuchObjectLockConfiguration: {
+		Code:           "NoSuchObjectLockConfiguration",
+		Description:    "The specified object does not have a ObjectLock configuration",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrObjectLocked: {
 		Code:           "InvalidRequest",
 		Description:    "Object is WORM protected and cannot be overwritten",
@@ -822,6 +866,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidTagDirective: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown tag directive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidEncryptionMethod: {
 		Code:           "InvalidRequest",
 		Description:    "The encryption method specified is not supported",
@@ -1037,6 +1086,11 @@ var errorCodes = errorCodeMap{
 		Description:    "A timeout occurred while trying to lock a resource",
 		HTTPStatusCode: http.StatusRequestTimeout,
 	},
+	ErrOperationMaxedOut: {
+		Code:           "XMinioServerTimedOut",
+		Description:    "A timeout exceeded while waiting to proceed with the request",
+		HTTPStatusCode: http.StatusRequestTimeout,
+	},
 	ErrUnsupportedMetadata: {
 		Code:           "InvalidArgument",
 		Description:    "Your metadata headers are not supported.",
@@ -1532,6 +1586,12 @@ var errorCodes = errorCodeMap{
 		Description:    "User is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrAddServiceAccountInvalidArgument: {
+		Code:           "XMinioInvalidArgument",
+		Description:    "New service accounts for admin access key is not allowed",
+		HTTPStatusCode: http.StatusConflict,
+	},
+
 	ErrPostPolicyConditionInvalidFormat: {
 		Code:           "PostPolicyInvalidKeyName",
 		Description:    "Invalid according to Policy: Policy Condition failed",
@@ -1600,18 +1660,20 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSNotConfigured
 	case crypto.ErrKMSAuthLogin:
 		apiErr = ErrKMSAuthFailure
-	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
+	case context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
 		apiErr = ErrSlowDown
-	case errInvalidRetentionDate:
+	case objectlock.ErrInvalidRetentionDate:
 		apiErr = ErrInvalidRetentionDate
-	case errPastObjectLockRetainDate:
+	case objectlock.ErrPastObjectLockRetainDate:
 		apiErr = ErrPastObjectLockRetainDate
-	case errUnknownWORMModeDirective:
+	case objectlock.ErrUnknownWORMModeDirective:
 		apiErr = ErrUnknownWORMModeDirective
-	case errObjectLockInvalidHeaders:
+	case objectlock.ErrObjectLockInvalidHeaders:
 		apiErr = ErrObjectLockInvalidHeaders
+	case objectlock.ErrMalformedXML:
+		apiErr = ErrMalformedXML
 	}
 
 	// Compression errors
@@ -1701,7 +1763,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case BucketPolicyNotFound:
 		apiErr = ErrNoSuchBucketPolicy
 	case BucketLifecycleNotFound:
-		apiErr = ErrNoSuchBucketLifecycle
+		apiErr = ErrNoSuchLifecycleConfiguration
+	case BucketSSEConfigNotFound:
+		apiErr = ErrNoSuchBucketSSEConfig
 	case *event.ErrInvalidEventName:
 		apiErr = ErrEventNotification
 	case *event.ErrInvalidARN:
@@ -1724,10 +1788,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrOverlappingFilterNotification
 	case *event.ErrUnsupportedConfiguration:
 		apiErr = ErrUnsupportedNotification
+	case OperationTimedOut:
+		apiErr = ErrOperationTimedOut
 	case BackendDown:
 		apiErr = ErrBackendDown
-	case crypto.Error:
-		apiErr = ErrObjectTampered
 	case ObjectNameTooLong:
 		apiErr = ErrKeyTooLongError
 	default:
@@ -1772,9 +1836,41 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// their internal error types. This code is only
 		// useful with gateway implementations.
 		switch e := err.(type) {
+		case *xml.SyntaxError:
+			apiErr = APIError{
+				Code: "MalformedXML",
+				Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description,
+					e.Error()),
+				HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
+			}
+		case url.EscapeError:
+			apiErr = APIError{
+				Code: "XMinioInvalidObjectName",
+				Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description,
+					e.Error()),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case lifecycle.Error:
+			apiErr = APIError{
+				Code:           "InvalidRequest",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case tagging.Error:
+			apiErr = APIError{
+				Code:           e.Code(),
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case policy.Error:
+			apiErr = APIError{
+				Code:           "MalformedPolicy",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case crypto.Error:
 			apiErr = APIError{
-				Code:           "XKMSInternalError",
+				Code:           "XMinIOEncryptionError",
 				Description:    e.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}

cmd/api-headers.go

@@ -22,7 +22,9 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/crypto"
@@ -80,6 +82,10 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 	}
 
+	if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
+		w.Header().Set(xhttp.AmzMpPartsCount, strconv.Itoa(len(objInfo.Parts)))
+	}
+
 	if objInfo.ContentType != "" {
 		w.Header().Set(xhttp.ContentType, objInfo.ContentType)
 	}
@@ -91,6 +97,17 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	if !objInfo.Expires.IsZero() {
 		w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
 	}
+	if globalCacheConfig.Enabled {
+		w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
+		w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
+	}
+
+	// Set tag count if object has tags
+	tags, _ := url.ParseQuery(objInfo.UserTags)
+	tagCount := len(tags)
+	if tagCount != 0 {
+		w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
+	}
 
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {

cmd/api-response.go

@@ -429,7 +429,12 @@ func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingTyp
 			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
-		content.StorageClass = object.StorageClass
+		if object.StorageClass != "" {
+			content.StorageClass = object.StorageClass
+		} else {
+			content.StorageClass = globalMinioDefaultStorageClass
+		}
+
 		content.Owner = owner
 		content.VersionID = "null"
 		content.IsLatest = true
@@ -475,7 +480,11 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
-		content.StorageClass = object.StorageClass
+		if object.StorageClass != "" {
+			content.StorageClass = object.StorageClass
+		} else {
+			content.StorageClass = globalMinioDefaultStorageClass
+		}
 		content.Owner = owner
 		contents = append(contents, content)
 	}
@@ -521,7 +530,11 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
-		content.StorageClass = object.StorageClass
+		if object.StorageClass != "" {
+			content.StorageClass = object.StorageClass
+		} else {
+			content.StorageClass = globalMinioDefaultStorageClass
+		}
 		content.Owner = owner
 		if metadata {
 			content.UserMetadata = make(StringMap)

cmd/api-router.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,9 +18,14 @@ package cmd
 
 import (
 	"net/http"
+	"strconv"
+	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/env"
 )
 
 func newHTTPServerFn() *xhttp.Server {
@@ -87,126 +92,229 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
+	requestsMax, _ := strconv.Atoi(env.Get(config.EnvAPIRequestsMax, "0"))
+	if requestsMax < 0 {
+		requestsMax = 0 // 0 means unlimited
+	}
+
+	var (
+		enabled              bool
+		requestsMaxPerServer int
+		requestsDeadline     time.Duration
+	)
+
+	if len(globalEndpoints.Hosts()) == 0 {
+		requestsMaxPerServer = requestsMax
+	} else {
+		requestsMaxPerServer = requestsMax / len(globalEndpoints.Hosts())
+	}
+
+	enabled = requestsMaxPerServer > 0
+	if enabled {
+		var err error
+		requestsDeadline, err = time.ParseDuration(env.Get(config.EnvAPIRequestsDeadline, "10s"))
+		if err != nil {
+			logger.Info("(%s) parsing environment value MINIO_API_REQUESTS_DEADLINE, defaulting to 10 seconds", err)
+			requestsDeadline = 10 * time.Second
+		}
+	}
+
+	requestsMaxCh := make(chan struct{}, requestsMaxPerServer)
+
 	for _, bucket := range routers {
 		// Object operations
 		// HeadObject
-		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler)))
+		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler)), enabled, requestsMaxCh, requestsDeadline))
 		// CopyObjectPart
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("listobjectparts", httpTraceAll(api.ListObjectPartsHandler))).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("listobjectparts", httpTraceAll(api.ListObjectPartsHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
-		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("completemutipartupload", httpTraceAll(api.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("completemutipartupload", httpTraceAll(api.CompleteMultipartUploadHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
-		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("newmultipartupload", httpTraceAll(api.NewMultipartUploadHandler))).Queries("uploads", "")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("newmultipartupload", httpTraceAll(api.NewMultipartUploadHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("uploads", "")
 		// AbortMultipartUpload
-		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler))).Queries("acl", "")
-		// GetObjectTagging - this is a dummy call.
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler))).Queries("tagging", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("acl", "")
+		// PutObjectACL - this is a dummy call.
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("putobjectacl", httpTraceHdrs(api.PutObjectACLHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("acl", "")
+		// GetObjectTagging
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("tagging", "")
+		// PutObjectTagging
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("putobjecttagging", httpTraceHdrs(api.PutObjectTaggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("tagging", "")
+		// DeleteObjectTagging
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("deleteobjecttagging", httpTraceHdrs(api.DeleteObjectTaggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("tagging", "")
 		// SelectObjectContent
-		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectretention", httpTraceHdrs(api.GetObjectRetentionHandler))).Queries("retention", "")
-		// GetObject
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler)))
-		// CopyObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler)))
-		// PutObjectRetention
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectretention", httpTraceHdrs(api.PutObjectRetentionHandler))).Queries("retention", "")
-		// PutObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler)))
-		// DeleteObject
-		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler)))
-		// PutObjectLegalHold
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectlegalhold", httpTraceHdrs(api.PutObjectLegalHoldHandler))).Queries("legal-hold", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("getobjectretention", httpTraceAll(api.GetObjectRetentionHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("retention", "")
 		// GetObjectLegalHold
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectlegalhold", httpTraceHdrs(api.GetObjectLegalHoldHandler))).Queries("legal-hold", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("getobjectlegalhold", httpTraceAll(api.GetObjectLegalHoldHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("legal-hold", "")
+		// GetObject
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler)), enabled, requestsMaxCh, requestsDeadline))
+		// CopyObject
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler)), enabled, requestsMaxCh, requestsDeadline))
+		// PutObjectRetention
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("retention", "")
+		// PutObjectLegalHold
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("putobjectlegalhold", httpTraceAll(api.PutObjectLegalHoldHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("legal-hold", "")
+
+		// PutObject
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler)), enabled, requestsMaxCh, requestsDeadline))
+		// DeleteObject
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler)), enabled, requestsMaxCh, requestsDeadline))
 
 		/// Bucket operations
 		// GetBucketLocation
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlocation", httpTraceAll(api.GetBucketLocationHandler))).Queries("location", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketlocation", httpTraceAll(api.GetBucketLocationHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("location", "")
 		// GetBucketPolicy
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler))).Queries("policy", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("policy", "")
 		// GetBucketLifecycle
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler))).Queries("lifecycle", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("lifecycle", "")
+		// GetBucketEncryption
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketencryption", httpTraceAll(api.GetBucketEncryptionHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("encryption", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler))).Queries("acl", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("acl", "")
+		// PutBucketACL -- this is a dummy call.
+		bucket.Methods(http.MethodPut).HandlerFunc(
+			maxClients(collectAPIStats("putbucketacl", httpTraceAll(api.PutBucketACLHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler))).Queries("cors", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketwebsite", httpTraceAll(api.GetBucketWebsiteHandler))).Queries("website", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketwebsite", httpTraceAll(api.GetBucketWebsiteHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("website", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketaccelerate", httpTraceAll(api.GetBucketAccelerateHandler))).Queries("accelerate", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketaccelerate", httpTraceAll(api.GetBucketAccelerateHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketrequestpayment", httpTraceAll(api.GetBucketRequestPaymentHandler))).Queries("requestPayment", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketrequestpayment", httpTraceAll(api.GetBucketRequestPaymentHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlogging", httpTraceAll(api.GetBucketLoggingHandler))).Queries("logging", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketlogging", httpTraceAll(api.GetBucketLoggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("logging", "")
 		// GetBucketLifecycleHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler))).Queries("lifecycle", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("lifecycle", "")
 		// GetBucketReplicationHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketreplication", httpTraceAll(api.GetBucketReplicationHandler))).Queries("replication", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketreplication", httpTraceAll(api.GetBucketReplicationHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("replication", "")
 		// GetBucketTaggingHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler))).Queries("tagging", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("tagging", "")
 		//DeleteBucketWebsiteHandler
-		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketwebsite", httpTraceAll(api.DeleteBucketWebsiteHandler))).Queries("website", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(
+			maxClients(collectAPIStats("deletebucketwebsite", httpTraceAll(api.DeleteBucketWebsiteHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("website", "")
 		// DeleteBucketTaggingHandler
-		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebuckettagging", httpTraceAll(api.DeleteBucketTaggingHandler))).Queries("tagging", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(
+			maxClients(collectAPIStats("deletebuckettagging", httpTraceAll(api.DeleteBucketTaggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("tagging", "")
 
 		// GetBucketObjectLockConfig
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketobjectlockconfiguration", httpTraceAll(api.GetBucketObjectLockConfigHandler))).Queries("object-lock", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketobjectlockconfiguration", httpTraceAll(api.GetBucketObjectLockConfigHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("object-lock", "")
 		// GetBucketVersioning
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketversioning", httpTraceAll(api.GetBucketVersioningHandler))).Queries("versioning", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketversioning", httpTraceAll(api.GetBucketVersioningHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("versioning", "")
 		// GetBucketNotification
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler))).Queries("notification", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("notification", "")
 		// ListenBucketNotification
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listenbucketnotification", httpTraceAll(api.ListenBucketNotificationHandler))).Queries("events", "{events:.*}")
 		// ListMultipartUploads
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listmultipartuploads", httpTraceAll(api.ListMultipartUploadsHandler))).Queries("uploads", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("listmultipartuploads", httpTraceAll(api.ListMultipartUploadsHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("uploads", "")
 		// ListObjectsV2M
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listobjectsv2M", httpTraceAll(api.ListObjectsV2MHandler))).Queries("list-type", "2", "metadata", "true")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("listobjectsv2M", httpTraceAll(api.ListObjectsV2MHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("list-type", "2", "metadata", "true")
 		// ListObjectsV2
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler))).Queries("list-type", "2")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler)), enabled, requestsMaxCh, requestsDeadline)).Queries("list-type", "2")
 		// ListBucketVersions
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listbucketversions", httpTraceAll(api.ListBucketObjectVersionsHandler))).Queries("versions", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("listbucketversions", httpTraceAll(api.ListBucketObjectVersionsHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("versions", "")
 		// ListObjectsV1 (Legacy)
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler)))
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler)), enabled, requestsMaxCh, requestsDeadline))
 		// PutBucketLifecycle
-		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler))).Queries("lifecycle", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(
+			maxClients(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("lifecycle", "")
+		// PutBucketEncryption
+		bucket.Methods(http.MethodPut).HandlerFunc(
+			maxClients(collectAPIStats("putbucketencryption", httpTraceAll(api.PutBucketEncryptionHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("encryption", "")
+
 		// PutBucketPolicy
-		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler))).Queries("policy", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(
+			maxClients(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("policy", "")
 
 		// PutBucketObjectLockConfig
-		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketobjectlockconfig", httpTraceAll(api.PutBucketObjectLockConfigHandler))).Queries("object-lock", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(
+			maxClients(collectAPIStats("putbucketobjectlockconfig", httpTraceAll(api.PutBucketObjectLockConfigHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("object-lock", "")
 		// PutBucketVersioning
-		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketversioning", httpTraceAll(api.PutBucketVersioningHandler))).Queries("versioning", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(
+			maxClients(collectAPIStats("putbucketversioning", httpTraceAll(api.PutBucketVersioningHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("versioning", "")
 		// PutBucketNotification
-		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketnotification", httpTraceAll(api.PutBucketNotificationHandler))).Queries("notification", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(
+			maxClients(collectAPIStats("putbucketnotification", httpTraceAll(api.PutBucketNotificationHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("notification", "")
 		// PutBucket
-		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucket", httpTraceAll(api.PutBucketHandler)))
+		bucket.Methods(http.MethodPut).HandlerFunc(
+			maxClients(collectAPIStats("putbucket", httpTraceAll(api.PutBucketHandler)), enabled, requestsMaxCh, requestsDeadline))
 		// HeadBucket
-		bucket.Methods(http.MethodHead).HandlerFunc(collectAPIStats("headbucket", httpTraceAll(api.HeadBucketHandler)))
+		bucket.Methods(http.MethodHead).HandlerFunc(
+			maxClients(collectAPIStats("headbucket", httpTraceAll(api.HeadBucketHandler)), enabled, requestsMaxCh, requestsDeadline))
 		// PostPolicy
-		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(collectAPIStats("postpolicybucket", httpTraceHdrs(api.PostPolicyBucketHandler)))
+		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(
+			maxClients(collectAPIStats("postpolicybucket", httpTraceHdrs(api.PostPolicyBucketHandler)), enabled, requestsMaxCh, requestsDeadline))
 		// DeleteMultipleObjects
-		bucket.Methods(http.MethodPost).HandlerFunc(collectAPIStats("deletemultipleobjects", httpTraceAll(api.DeleteMultipleObjectsHandler))).Queries("delete", "")
+		bucket.Methods(http.MethodPost).HandlerFunc(
+			maxClients(collectAPIStats("deletemultipleobjects", httpTraceAll(api.DeleteMultipleObjectsHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("delete", "")
 		// DeleteBucketPolicy
-		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler))).Queries("policy", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(
+			maxClients(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("policy", "")
 		// DeleteBucketLifecycle
-		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler))).Queries("lifecycle", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(
+			maxClients(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("lifecycle", "")
+		// DeleteBucketEncryption
+		bucket.Methods(http.MethodDelete).HandlerFunc(
+			maxClients(collectAPIStats("deletebucketencryption", httpTraceAll(api.DeleteBucketEncryptionHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("encryption", "")
 		// DeleteBucket
-		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler)))
+		bucket.Methods(http.MethodDelete).HandlerFunc(
+			maxClients(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler)), enabled, requestsMaxCh, requestsDeadline))
 	}
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler)))
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
+		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler)), enabled, requestsMaxCh, requestsDeadline))
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = http.HandlerFunc(collectAPIStats("notfound", httpTraceAll(errorResponseHandler)))

cmd/auth-handler.go

@@ -26,15 +26,18 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"strings"
+	"time"
 
-	jwtgo "github.com/dgrijalva/jwt-go"
 	xhttp "github.com/minio/minio/cmd/http"
+	xjwt "github.com/minio/minio/cmd/jwt"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/hash"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/policy"
 )
 
 // Verify if request has JWT.
@@ -179,12 +182,14 @@ func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
 
 // Fetch claims in the security token returned by the client.
 func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
-	claims := make(map[string]interface{})
+	claims := xjwt.NewMapClaims()
+
 	token := getSessionToken(r)
 	if token == "" {
-		return claims, nil
+		return claims.Map(), nil
 	}
-	stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
+
+	stsTokenCallback := func(claims *xjwt.MapClaims) ([]byte, error) {
 		// JWT token for x-amz-security-token is signed with admin
 		// secret key, temporary credentials become invalid if
 		// server admin credentials change. This is done to ensure
@@ -195,66 +200,43 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 		// on the client side and is treated like an opaque value.
 		return []byte(globalActiveCred.SecretKey), nil
 	}
-	p := &jwtgo.Parser{
-		ValidMethods: []string{
-			jwtgo.SigningMethodHS256.Alg(),
-			jwtgo.SigningMethodHS512.Alg(),
-		},
-	}
-	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
-	if err != nil {
-		return nil, err
-	}
-	if !jtoken.Valid {
+
+	if err := xjwt.ParseWithClaims(token, claims, stsTokenCallback); err != nil {
 		return nil, errAuthentication
 	}
-	v, ok := claims["accessKey"]
-	if !ok {
-		return nil, errInvalidAccessKeyID
-	}
-	if _, ok = v.(string); !ok {
-		return nil, errInvalidAccessKeyID
-	}
 
 	if globalPolicyOPA == nil {
-		// If OPA is not set and if ldap claim key is set,
-		// allow the claim.
-		if _, ok := claims[ldapUser]; ok {
-			return claims, nil
+		// If OPA is not set and if ldap claim key is set, allow the claim.
+		if _, ok := claims.Lookup(ldapUser); ok {
+			return claims.Map(), nil
 		}
 
 		// If OPA is not set, session token should
 		// have a policy and its mandatory, reject
 		// requests without policy claim.
-		p, pok := claims[iamPolicyName()]
-		if !pok {
+		_, pokOpenID := claims.Lookup(iamPolicyClaimNameOpenID())
+		_, pokSA := claims.Lookup(iamPolicyClaimNameSA())
+		if !pokOpenID && !pokSA {
 			return nil, errAuthentication
 		}
-		if _, pok = p.(string); !pok {
-			return nil, errAuthentication
-		}
-		sp, spok := claims[iampolicy.SessionPolicyName]
-		// Sub policy is optional, if not set return success.
+
+		sp, spok := claims.Lookup(iampolicy.SessionPolicyName)
 		if !spok {
-			return claims, nil
-		}
-		// Sub policy is set but its not a string, reject such requests
-		spStr, spok := sp.(string)
-		if !spok {
-			return nil, errAuthentication
+			return claims.Map(), nil
 		}
 		// Looks like subpolicy is set and is a string, if set then its
 		// base64 encoded, decode it. Decoding fails reject such requests.
-		spBytes, err := base64.StdEncoding.DecodeString(spStr)
+		spBytes, err := base64.StdEncoding.DecodeString(sp)
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
 			// something is malforming the request sent by client.
-			logger.LogIf(context.Background(), err, logger.Application)
+			logger.LogIf(r.Context(), err, logger.Application)
 			return nil, errAuthentication
 		}
-		claims[iampolicy.SessionPolicyName] = string(spBytes)
+		claims.MapClaims[iampolicy.SessionPolicyName] = string(spBytes)
 	}
-	return claims, nil
+
+	return claims.Map(), nil
 }
 
 // Fetch claims in the security token returned by the client and validate the token.
@@ -268,7 +250,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 	}
 	claims, err := getClaimsFromToken(r)
 	if err != nil {
-		return nil, toAPIErrorCode(context.Background(), err)
+		return nil, toAPIErrorCode(r.Context(), err)
 	}
 	return claims, ErrNone
 }
@@ -293,7 +275,7 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 	var cred auth.Credentials
 	switch getRequestAuthType(r) {
 	case authTypeUnknown, authTypeStreamingSigned:
-		return accessKey, owner, ErrAccessDenied
+		return accessKey, owner, ErrSignatureVersionNotSupported
 	case authTypePresignedV2, authTypeSignedV2:
 		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
 			return accessKey, owner, s3Err
@@ -355,7 +337,7 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 			// Request is allowed return the appropriate access key.
 			return cred.AccessKey, owner, ErrNone
 		}
-		return accessKey, owner, ErrAccessDenied
+		return cred.AccessKey, owner, ErrAccessDenied
 	}
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
@@ -369,7 +351,7 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Request is allowed return the appropriate access key.
 		return cred.AccessKey, owner, ErrNone
 	}
-	return accessKey, owner, ErrAccessDenied
+	return cred.AccessKey, owner, ErrAccessDenied
 }
 
 // Verify if request has valid AWS Signature Version '2'.
@@ -402,12 +384,11 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 		err                       error
 		contentMD5, contentSHA256 []byte
 	)
+
 	// Extract 'Content-Md5' if present.
-	if _, ok := r.Header[xhttp.ContentMD5]; ok {
-		contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get(xhttp.ContentMD5))
-		if err != nil || len(contentMD5) == 0 {
-			return ErrInvalidDigest
-		}
+	contentMD5, err = checkValidMD5(r.Header)
+	if err != nil {
+		return ErrInvalidDigest
 	}
 
 	// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
@@ -483,7 +464,107 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		a.handler.ServeHTTP(w, r)
 		return
 	}
-	writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
+	writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
+}
+
+func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool, map[string]interface{}, APIErrorCode) {
+	var cred auth.Credentials
+	var owner bool
+	var s3Err APIErrorCode
+	switch atype {
+	case authTypeUnknown, authTypeStreamingSigned:
+		return cred, owner, nil, ErrSignatureVersionNotSupported
+	case authTypeSignedV2, authTypePresignedV2:
+		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
+			return cred, owner, nil, s3Err
+		}
+		cred, owner, s3Err = getReqAccessKeyV2(r)
+	case authTypePresigned, authTypeSigned:
+		region := globalServerRegion
+		if s3Err = isReqAuthenticated(GlobalContext, r, region, serviceS3); s3Err != ErrNone {
+			return cred, owner, nil, s3Err
+		}
+		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
+	}
+	if s3Err != ErrNone {
+		return cred, owner, nil, s3Err
+	}
+
+	claims, s3Err := checkClaimsFromToken(r, cred)
+	if s3Err != ErrNone {
+		return cred, owner, nil, s3Err
+	}
+
+	return cred, owner, claims, ErrNone
+}
+
+func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool, claims map[string]interface{}) (s3Err APIErrorCode) {
+	var retSet bool
+	if cred.AccessKey == "" {
+		conditions := getConditionValues(r, "", "", nil)
+		conditions["object-lock-mode"] = []string{string(retMode)}
+		conditions["object-lock-retain-until-date"] = []string{retDate.Format(time.RFC3339)}
+		if retDays > 0 {
+			conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
+		}
+		if retMode == objectlock.RetGovernance && byPassSet {
+			byPassSet = globalPolicySys.IsAllowed(policy.Args{
+				AccountName:     cred.AccessKey,
+				Action:          policy.Action(policy.BypassGovernanceRetentionAction),
+				BucketName:      bucketName,
+				ConditionValues: conditions,
+				IsOwner:         false,
+				ObjectName:      objectName,
+			})
+		}
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          policy.Action(policy.PutObjectRetentionAction),
+			BucketName:      bucketName,
+			ConditionValues: conditions,
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			retSet = true
+		}
+		if byPassSet || retSet {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	conditions := getConditionValues(r, "", cred.AccessKey, claims)
+	conditions["object-lock-mode"] = []string{string(retMode)}
+	conditions["object-lock-retain-until-date"] = []string{retDate.Format(time.RFC3339)}
+	if retDays > 0 {
+		conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
+	}
+	if retMode == objectlock.RetGovernance && byPassSet {
+		byPassSet = globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          policy.BypassGovernanceRetentionAction,
+			BucketName:      bucketName,
+			ObjectName:      objectName,
+			ConditionValues: conditions,
+			IsOwner:         owner,
+			Claims:          claims,
+		})
+	}
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
+		Action:          policy.PutObjectRetentionAction,
+		BucketName:      bucketName,
+		ConditionValues: conditions,
+		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
+	}) {
+		retSet = true
+	}
+	if byPassSet || retSet {
+		return ErrNone
+	}
+	return ErrAccessDenied
 }
 
 // isPutActionAllowed - check if PUT operation is allowed on the resource, this
@@ -494,7 +575,7 @@ func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.R
 	var owner bool
 	switch atype {
 	case authTypeUnknown:
-		return ErrAccessDenied
+		return ErrSignatureVersionNotSupported
 	case authTypeSignedV2, authTypePresignedV2:
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
@@ -513,7 +594,7 @@ func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.R
 	if cred.AccessKey == "" {
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          policy.PutObjectAction,
+			Action:          policy.Action(action),
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", "", nil),
 			IsOwner:         false,

cmd/background-heal-ops.go

@@ -52,43 +52,48 @@ func (h *healRoutine) queueHealTask(task healTask) {
 	h.tasks <- task
 }
 
+func waitForLowHTTPReq(tolerance int32) {
+	if httpServer := newHTTPServerFn(); httpServer != nil {
+		// Wait at max 10 minute for an inprogress request before proceeding to heal
+		waitCount := 600
+		// Any requests in progress, delay the heal.
+		for (httpServer.GetRequestCount() >= tolerance) &&
+			waitCount > 0 {
+			waitCount--
+			time.Sleep(1 * time.Second)
+		}
+	}
+}
+
 // Wait for heal requests and process them
-func (h *healRoutine) run() {
-	ctx := context.Background()
+func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 	for {
 		select {
 		case task, ok := <-h.tasks:
 			if !ok {
 				break
 			}
-			if httpServer := newHTTPServerFn(); httpServer != nil {
-				// Wait at max 10 minute for an inprogress request before proceeding to heal
-				waitCount := 600
-				// Any requests in progress, delay the heal.
-				for (httpServer.GetRequestCount() >= int32(globalEndpoints.Nodes())) &&
-					waitCount > 0 {
-					waitCount--
-					time.Sleep(1 * time.Second)
-				}
-			}
+
+			// Wait and proceed if there are active requests
+			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()))
 
 			var res madmin.HealResultItem
 			var err error
-			bucket, object := urlPath2BucketObjectName(task.path)
+			bucket, object := path2BucketObject(task.path)
 			switch {
 			case bucket == "" && object == "":
-				res, err = bgHealDiskFormat(ctx, task.opts)
+				res, err = healDiskFormat(ctx, objAPI, task.opts)
 			case bucket != "" && object == "":
-				res, err = bgHealBucket(ctx, bucket, task.opts)
+				res, err = objAPI.HealBucket(ctx, bucket, task.opts.DryRun, task.opts.Remove)
 			case bucket != "" && object != "":
-				res, err = bgHealObject(ctx, bucket, object, task.opts)
+				res, err = objAPI.HealObject(ctx, bucket, object, task.opts)
 			}
 			if task.responseCh != nil {
 				task.responseCh <- healResult{result: res, err: err}
 			}
 		case <-h.doneCh:
 			return
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			return
 		}
 	}
@@ -102,45 +107,27 @@ func initHealRoutine() *healRoutine {
 
 }
 
-func startBackgroundHealing() {
-	ctx := context.Background()
-
-	var objAPI ObjectLayer
-	for {
-		objAPI = newObjectLayerWithoutSafeModeFn()
-		if objAPI == nil {
-			time.Sleep(time.Second)
-			continue
-		}
-		break
-	}
-
+func startBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	// Run the background healer
 	globalBackgroundHealRoutine = initHealRoutine()
-	go globalBackgroundHealRoutine.run()
+	go globalBackgroundHealRoutine.run(ctx, objAPI)
 
 	// Launch the background healer sequence to track
 	// background healing operations
-	info := objAPI.StorageInfo(ctx)
+	info := objAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
 	nh := newBgHealSequence(numDisks)
 	globalBackgroundHealState.LaunchNewHealSequence(nh)
 }
 
-func initBackgroundHealing() {
-	go startBackgroundHealing()
+func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+	go startBackgroundHealing(ctx, objAPI)
 }
 
-// bgHealDiskFormat - heals format.json, return value indicates if a
+// healDiskFormat - heals format.json, return value indicates if a
 // failure error occurred.
-func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealResultItem, error) {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
-		return madmin.HealResultItem{}, errServerNotInitialized
-	}
-
-	res, err := objectAPI.HealFormat(ctx, opts.DryRun)
+func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpts) (madmin.HealResultItem, error) {
+	res, err := objAPI.HealFormat(ctx, opts.DryRun)
 
 	// return any error, ignore error returned when disks have
 	// already healed.
@@ -161,24 +148,3 @@ func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealRes
 
 	return res, nil
 }
-
-// bghealBucket - traverses and heals given bucket
-func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
-		return madmin.HealResultItem{}, errServerNotInitialized
-	}
-
-	return objectAPI.HealBucket(ctx, bucket, opts.DryRun, opts.Remove)
-}
-
-// bgHealObject - heal the given object and record result
-func bgHealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
-		return madmin.HealResultItem{}, errServerNotInitialized
-	}
-	return objectAPI.HealObject(ctx, bucket, object, opts.DryRun, opts.Remove, opts.ScanMode)
-}

cmd/background-newdisks-heal-ops.go

@@ -25,32 +25,19 @@ import (
 
 const defaultMonitorNewDiskInterval = time.Minute * 10
 
-func initLocalDisksAutoHeal() {
-	go monitorLocalDisksAndHeal()
+func initLocalDisksAutoHeal(ctx context.Context, objAPI ObjectLayer) {
+	go monitorLocalDisksAndHeal(ctx, objAPI)
 }
 
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal() {
-	// Wait until the object layer is ready
-	var objAPI ObjectLayer
-	for {
-		objAPI = newObjectLayerWithoutSafeModeFn()
-		if objAPI == nil {
-			time.Sleep(time.Second)
-			continue
-		}
-		break
-	}
-
+func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 	z, ok := objAPI.(*xlZones)
 	if !ok {
 		return
 	}
 
-	ctx := context.Background()
-
 	var bgSeq *healSequence
 	var found bool
 
@@ -62,64 +49,69 @@ func monitorLocalDisksAndHeal() {
 		time.Sleep(time.Second)
 	}
 
-	// Perform automatic disk healing when a new one is inserted
+	// Perform automatic disk healing when a disk is replaced locally.
 	for {
-		time.Sleep(defaultMonitorNewDiskInterval)
-
-		localDisksInZoneHeal := make([]Endpoints, len(z.zones))
-		for i, ep := range globalEndpoints {
-			localDisksToHeal := Endpoints{}
-			for _, endpoint := range ep.Endpoints {
-				if !endpoint.IsLocal {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(defaultMonitorNewDiskInterval):
+			// Attempt a heal as the server starts-up first.
+			localDisksInZoneHeal := make([]Endpoints, len(z.zones))
+			for i, ep := range globalEndpoints {
+				localDisksToHeal := Endpoints{}
+				for _, endpoint := range ep.Endpoints {
+					if !endpoint.IsLocal {
+						continue
+					}
+					// Try to connect to the current endpoint
+					// and reformat if the current disk is not formatted
+					_, _, err := connectEndpoint(endpoint)
+					if err == errUnformattedDisk {
+						localDisksToHeal = append(localDisksToHeal, endpoint)
+					}
+				}
+				if len(localDisksToHeal) == 0 {
 					continue
 				}
-				// Try to connect to the current endpoint
-				// and reformat if the current disk is not formatted
-				_, _, err := connectEndpoint(endpoint)
-				if err == errUnformattedDisk {
-					localDisksToHeal = append(localDisksToHeal, endpoint)
-				}
+				localDisksInZoneHeal[i] = localDisksToHeal
 			}
-			if len(localDisksToHeal) == 0 {
-				continue
-			}
-			localDisksInZoneHeal[i] = localDisksToHeal
-		}
 
-		// Reformat disks
-		bgSeq.sourceCh <- SlashSeparator
-		// Ensure that reformatting disks is finished
-		bgSeq.sourceCh <- nopHeal
+			// Reformat disks
+			bgSeq.sourceCh <- healSource{path: SlashSeparator}
 
-		var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
-		// Compute the list of erasure set to heal
-		for i, localDisksToHeal := range localDisksInZoneHeal {
-			var erasureSetToHeal []int
-			for _, endpoint := range localDisksToHeal {
-				// Load the new format of this passed endpoint
-				_, format, err := connectEndpoint(endpoint)
-				if err != nil {
-					logger.LogIf(ctx, err)
-					continue
+			// Ensure that reformatting disks is finished
+			bgSeq.sourceCh <- healSource{path: nopHeal}
+
+			var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
+			// Compute the list of erasure set to heal
+			for i, localDisksToHeal := range localDisksInZoneHeal {
+				var erasureSetToHeal []int
+				for _, endpoint := range localDisksToHeal {
+					// Load the new format of this passed endpoint
+					_, format, err := connectEndpoint(endpoint)
+					if err != nil {
+						logger.LogIf(ctx, err)
+						continue
+					}
+					// Calculate the set index where the current endpoint belongs
+					setIndex, _, err := findDiskIndex(z.zones[i].format, format)
+					if err != nil {
+						logger.LogIf(ctx, err)
+						continue
+					}
+
+					erasureSetToHeal = append(erasureSetToHeal, setIndex)
 				}
-				// Calculate the set index where the current endpoint belongs
-				setIndex, _, err := findDiskIndex(z.zones[i].format, format)
-				if err != nil {
-					logger.LogIf(ctx, err)
-					continue
-				}
-
-				erasureSetToHeal = append(erasureSetToHeal, setIndex)
+				erasureSetInZoneToHeal[i] = erasureSetToHeal
 			}
-			erasureSetInZoneToHeal[i] = erasureSetToHeal
-		}
 
-		// Heal all erasure sets that need
-		for i, erasureSetToHeal := range erasureSetInZoneToHeal {
-			for _, setIndex := range erasureSetToHeal {
-				err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex])
-				if err != nil {
-					logger.LogIf(ctx, err)
+			// Heal all erasure sets that need
+			for i, erasureSetToHeal := range erasureSetInZoneToHeal {
+				for _, setIndex := range erasureSetToHeal {
+					err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex])
+					if err != nil {
+						logger.LogIf(ctx, err)
+					}
 				}
 			}
 		}

cmd/bitrot-streaming.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -27,6 +26,14 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
+type errHashMismatch struct {
+	message string
+}
+
+func (err *errHashMismatch) Error() string {
+	return err.message
+}
+
 // Calculates bitrot in chunks and writes the hash into the stream.
 type streamingBitrotWriter struct {
 	iow       *io.PipeWriter
@@ -132,9 +139,9 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err = fmt.Errorf("hashes do not match expected %s, got %s",
-			hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))
-		logger.LogIf(context.Background(), err)
+		err := &errHashMismatch{fmt.Sprintf("hashes do not match expected %s, got %s",
+			hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
+		logger.LogIf(GlobalContext, err)
 		return 0, err
 	}
 	b.currOffset += int64(len(buf))

cmd/bitrot-whole.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"hash"
 	"io"
 
@@ -36,12 +35,12 @@ type wholeBitrotWriter struct {
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
 	err := b.disk.AppendFile(b.volume, b.filePath, p)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(GlobalContext, err)
 		return 0, err
 	}
 	_, err = b.Hash.Write(p)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(GlobalContext, err)
 		return 0, err
 	}
 	return len(p), nil
@@ -70,14 +69,14 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
 		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			ctx := context.Background()
+			ctx := GlobalContext
 			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
 			logger.LogIf(ctx, err)
 			return 0, err
 		}
 	}
 	if len(b.buf) < len(buf) {
-		logger.LogIf(context.Background(), errLessData)
+		logger.LogIf(GlobalContext, errLessData)
 		return 0, errLessData
 	}
 	n = copy(buf, b.buf)

cmd/bitrot.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"errors"
 	"hash"
 	"io"
@@ -72,7 +71,7 @@ func (a BitrotAlgorithm) New() hash.Hash {
 		hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
 		return hh
 	default:
-		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
+		logger.CriticalIf(GlobalContext, errors.New("Unsupported bitrot algorithm"))
 		return nil
 	}
 }
@@ -88,7 +87,7 @@ func (a BitrotAlgorithm) Available() bool {
 func (a BitrotAlgorithm) String() string {
 	name, ok := bitrotAlgorithms[a]
 	if !ok {
-		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
+		logger.CriticalIf(GlobalContext, errors.New("Unsupported bitrot algorithm"))
 	}
 	return name
 }

cmd/bootstrap-peer-server.go

@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
@@ -62,9 +62,9 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 		return fmt.Errorf("Expected platform '%s', found to be running '%s'",
 			s1.MinioPlatform, s2.MinioPlatform)
 	}
-	if s1.MinioEndpoints.Nodes() != s2.MinioEndpoints.Nodes() {
-		return fmt.Errorf("Expected number of endpoints %d, seen %d", s1.MinioEndpoints.Nodes(),
-			s2.MinioEndpoints.Nodes())
+	if s1.MinioEndpoints.NEndpoints() != s2.MinioEndpoints.NEndpoints() {
+		return fmt.Errorf("Expected number of endpoints %d, seen %d", s1.MinioEndpoints.NEndpoints(),
+			s2.MinioEndpoints.NEndpoints())
 	}
 
 	for i, ep := range s1.MinioEndpoints {
@@ -110,7 +110,7 @@ func registerBootstrapRESTHandlers(router *mux.Router) {
 		httpTraceHdrs(server.VerifyHandler))
 }
 
-// client to talk to bootstrap Nodes.
+// client to talk to bootstrap NEndpoints.
 type bootstrapRESTClient struct {
 	endpoint   Endpoint
 	restClient *rest.Client
@@ -126,7 +126,7 @@ func (client *bootstrapRESTClient) reConnect() {
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *bootstrapRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	return client.callWithContext(context.Background(), method, values, body, length)
+	return client.callWithContext(GlobalContext, method, values, body, length)
 }
 
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected

cmd/bucket-encryption-handlers.go

@@ -0,0 +1,181 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/policy"
+)
+
+const (
+	// Bucket Encryption configuration file name.
+	bucketSSEConfig = "bucket-encryption.xml"
+)
+
+// PutBucketEncryptionHandler - Stores given bucket encryption configuration
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketEncryption.html
+func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketEncryption")
+
+	defer logger.AuditLog(w, r, "PutBucketEncryption", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// PutBucketEncyrption API requires Content-Md5
+	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Parse bucket encryption xml
+	encConfig, err := validateBucketSSEConfig(io.LimitReader(r.Body, maxBucketSSEConfigSize))
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Return error if KMS is not initialized
+	if GlobalKMS == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Store the bucket encryption configuration in the object layer
+	if err = objAPI.SetBucketSSEConfig(ctx, bucket, encConfig); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Update the in-memory bucket encryption cache
+	globalBucketSSEConfigSys.Set(bucket, *encConfig)
+
+	// Update peer MinIO servers of the updated bucket encryption config
+	globalNotificationSys.SetBucketSSEConfig(ctx, bucket, encConfig)
+
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketEncryptionHandler - Returns bucket policy configuration
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketEncryption.html
+func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketEncryption")
+
+	defer logger.AuditLog(w, r, "GetBucketEncryption", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists
+	var err error
+	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Fetch bucket encryption configuration from object layer
+	var encConfig *bucketsse.BucketSSEConfig
+	if encConfig, err = objAPI.GetBucketSSEConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	var encConfigData []byte
+	if encConfigData, err = xml.Marshal(encConfig); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write bucket encryption configuration to client
+	writeSuccessResponseXML(w, encConfigData)
+}
+
+// DeleteBucketEncryptionHandler - Removes bucket encryption configuration
+func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteBucketEncryption")
+
+	defer logger.AuditLog(w, r, "DeleteBucketEncryption", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists
+	var err error
+	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Delete bucket encryption config from object layer
+	if err = objAPI.DeleteBucketSSEConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Remove entry from the in-memory bucket encryption cache
+	globalBucketSSEConfigSys.Remove(bucket)
+	// Update peer MinIO servers of the updated bucket encryption config
+	globalNotificationSys.RemoveBucketSSEConfig(ctx, bucket)
+
+	writeSuccessNoContent(w)
+}

cmd/bucket-encryption.go

@@ -0,0 +1,169 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/xml"
+	"errors"
+	"io"
+	"path"
+	"sync"
+
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+)
+
+// BucketSSEConfigSys - in-memory cache of bucket encryption config
+type BucketSSEConfigSys struct {
+	sync.RWMutex
+	bucketSSEConfigMap map[string]bucketsse.BucketSSEConfig
+}
+
+// NewBucketSSEConfigSys - Creates an empty in-memory bucket encryption configuration cache
+func NewBucketSSEConfigSys() *BucketSSEConfigSys {
+	return &BucketSSEConfigSys{
+		bucketSSEConfigMap: make(map[string]bucketsse.BucketSSEConfig),
+	}
+}
+
+// load - Loads the bucket encryption configuration for the given list of buckets
+func (sys *BucketSSEConfigSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
+	for _, bucket := range buckets {
+		config, err := objAPI.GetBucketSSEConfig(GlobalContext, bucket.Name)
+		if err != nil {
+			if _, ok := err.(BucketSSEConfigNotFound); ok {
+				sys.Remove(bucket.Name)
+			}
+			continue
+		}
+		sys.Set(bucket.Name, *config)
+	}
+
+	return nil
+}
+
+// Init - Initializes in-memory bucket encryption config cache for the given list of buckets
+func (sys *BucketSSEConfigSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// We don't cache bucket encryption config in gateway mode, nothing to do.
+	if globalIsGateway {
+		return nil
+	}
+
+	// Load bucket encryption config cache once during boot.
+	return sys.load(buckets, objAPI)
+}
+
+// Get - gets bucket encryption config for the given bucket.
+func (sys *BucketSSEConfigSys) Get(bucket string) (config bucketsse.BucketSSEConfig, ok bool) {
+	// We don't cache bucket encryption config in gateway mode.
+	if globalIsGateway {
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			return
+		}
+
+		cfg, err := objAPI.GetBucketSSEConfig(GlobalContext, bucket)
+		if err != nil {
+			return
+		}
+		return *cfg, true
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+	config, ok = sys.bucketSSEConfigMap[bucket]
+	return
+}
+
+// Set - sets bucket encryption config to given bucket name.
+func (sys *BucketSSEConfigSys) Set(bucket string, config bucketsse.BucketSSEConfig) {
+	// We don't cache bucket encryption config in gateway mode.
+	if globalIsGateway {
+		return
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+	sys.bucketSSEConfigMap[bucket] = config
+}
+
+// Remove - removes bucket encryption config for given bucket.
+func (sys *BucketSSEConfigSys) Remove(bucket string) {
+	sys.Lock()
+	defer sys.Unlock()
+
+	delete(sys.bucketSSEConfigMap, bucket)
+}
+
+// saveBucketSSEConfig - save bucket encryption config for given bucket.
+func saveBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string, config *bucketsse.BucketSSEConfig) error {
+	data, err := xml.Marshal(config)
+	if err != nil {
+		return err
+	}
+
+	// Path to store bucket encryption config for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
+	return saveConfig(ctx, objAPI, configFile, data)
+}
+
+// getBucketSSEConfig - get bucket encryption config for given bucket.
+func getBucketSSEConfig(objAPI ObjectLayer, bucket string) (*bucketsse.BucketSSEConfig, error) {
+	// Path to bucket-encryption.xml for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
+	configData, err := readConfig(GlobalContext, objAPI, configFile)
+	if err != nil {
+		if err == errConfigNotFound {
+			err = BucketSSEConfigNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+
+	return bucketsse.ParseBucketSSEConfig(bytes.NewReader(configData))
+}
+
+// removeBucketSSEConfig - removes bucket encryption config for given bucket.
+func removeBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error {
+	// Path to bucket-encryption.xml for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
+
+	if err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile); err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			return BucketSSEConfigNotFound{Bucket: bucket}
+		}
+		return err
+	}
+	return nil
+}
+
+// validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.
+func validateBucketSSEConfig(r io.Reader) (*bucketsse.BucketSSEConfig, error) {
+	encConfig, err := bucketsse.ParseBucketSSEConfig(r)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(encConfig.Rules) == 1 && encConfig.Rules[0].DefaultEncryptionAction.Algorithm == bucketsse.AES256 {
+		return encConfig, nil
+	}
+	return nil, errors.New("Unsupported bucket encryption configuration")
+}

cmd/bucket-encryption_test.go

@@ -0,0 +1,70 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"testing"
+)
+
+func TestValidateBucketSSEConfig(t *testing.T) {
+	testCases := []struct {
+		inputXML    string
+		expectedErr error
+		shouldPass  bool
+	}{
+		// MinIO supported XML
+		{
+			inputXML: `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+			<Rule>
+			<ApplyServerSideEncryptionByDefault>
+			<SSEAlgorithm>AES256</SSEAlgorithm>
+			</ApplyServerSideEncryptionByDefault>
+			</Rule>
+			</ServerSideEncryptionConfiguration>`,
+			expectedErr: nil,
+			shouldPass:  true,
+		},
+		// Unsupported XML
+		{
+			inputXML: `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+			<Rule>
+			<ApplyServerSideEncryptionByDefault>
+                        <SSEAlgorithm>aws:kms</SSEAlgorithm>
+                        <KMSMasterKeyID>arn:aws:kms:us-east-1:1234/5678example</KMSMasterKeyID>
+			</ApplyServerSideEncryptionByDefault>
+			</Rule>
+			</ServerSideEncryptionConfiguration>`,
+			expectedErr: errors.New("Unsupported bucket encryption configuration"),
+			shouldPass:  false,
+		},
+	}
+
+	for i, tc := range testCases {
+		_, err := validateBucketSSEConfig(bytes.NewReader([]byte(tc.inputXML)))
+		if tc.shouldPass && err != nil {
+			t.Fatalf("Test case %d: Expected to succeed but got %s", i+1, err)
+		}
+
+		if !tc.shouldPass {
+			if err == nil || err != nil && err.Error() != tc.expectedErr.Error() {
+				t.Fatalf("Test case %d: Expected %s but got %s", i+1, tc.expectedErr, err)
+			}
+		}
+	}
+}

cmd/bucket-handlers-listobjects.go

@@ -24,7 +24,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 // Validate all the ListObjects query arguments, returns an APIErrorCode

cmd/bucket-handlers.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2015, 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2015-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,12 +17,10 @@
 package cmd
 
 import (
-	"context"
 	"encoding/base64"
 	"encoding/xml"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"path"
@@ -36,11 +34,12 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
@@ -70,83 +69,78 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	// Get buckets in the DNS
 	dnsBuckets, err := globalDNSConfig.List()
 	if err != nil && err != dns.ErrNoEntriesFound {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(GlobalContext, err)
 		return
 	}
 
-	bucketSet := set.NewStringSet()
+	bucketsSet := set.NewStringSet()
+	bucketsToBeUpdated := set.NewStringSet()
+	bucketsInConflict := set.NewStringSet()
+	for _, bucket := range buckets {
+		bucketsSet.Add(bucket.Name)
+		r, ok := dnsBuckets[bucket.Name]
+		if !ok {
+			bucketsToBeUpdated.Add(bucket.Name)
+			continue
+		}
+		if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+			if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				// No difference in terms of domainIPs and nothing
+				// has changed so we don't change anything on the etcd.
+				continue
+			}
+			// if domain IPs intersect then it won't be an empty set.
+			// such an intersection means that bucket exists on etcd.
+			// but if we do see a difference with local domain IPs with
+			// hostSlice from etcd then we should update with newer
+			// domainIPs, we proceed to do that here.
+			bucketsToBeUpdated.Add(bucket.Name)
+			continue
+		}
+		// No IPs seem to intersect, this means that bucket exists but has
+		// different IP addresses perhaps from a different deployment.
+		// bucket names are globally unique in federation at a given
+		// path prefix, name collision is not allowed. We simply log
+		// an error and continue.
+		bucketsInConflict.Add(bucket.Name)
+	}
 
-	// Add buckets that are not registered with the DNS
+	// Add/update buckets that are not registered with the DNS
 	g := errgroup.WithNErrs(len(buckets))
-	for index := range buckets {
-		bucketSet.Add(buckets[index].Name)
+	bucketsToBeUpdatedSlice := bucketsToBeUpdated.ToSlice()
+	for index := range bucketsToBeUpdatedSlice {
 		index := index
 		g.Go(func() error {
-			r, gerr := globalDNSConfig.Get(buckets[index].Name)
-			if gerr != nil {
-				if gerr == dns.ErrNoEntriesFound {
-					return globalDNSConfig.Put(buckets[index].Name)
-				}
-				return gerr
-			}
-			if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-					// No difference in terms of domainIPs and nothing
-					// has changed so we don't change anything on the etcd.
-					return nil
-				}
-				// if domain IPs intersect then it won't be an empty set.
-				// such an intersection means that bucket exists on etcd.
-				// but if we do see a difference with local domain IPs with
-				// hostSlice from etcd then we should update with newer
-				// domainIPs, we proceed to do that here.
-				return globalDNSConfig.Put(buckets[index].Name)
-			}
-			// No IPs seem to intersect, this means that bucket exists but has
-			// different IP addresses perhaps from a different deployment.
-			// bucket names are globally unique in federation at a given
-			// path prefix, name collision is not allowed. We simply log
-			// an error and continue.
-			return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", buckets[index].Name, globalDomainIPs.ToSlice())
+			return globalDNSConfig.Put(bucketsToBeUpdatedSlice[index])
 		}, index)
 	}
 
 	for _, err := range g.Wait() {
 		if err != nil {
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(GlobalContext, err)
 		}
 	}
 
-	g = errgroup.WithNErrs(len(dnsBuckets))
-	// Remove buckets that are in DNS for this server, but aren't local
-	for index := range dnsBuckets {
-		index := index
-		g.Go(func() error {
-			// This is a local bucket that exists, so we can continue
-			if bucketSet.Contains(dnsBuckets[index].Key) {
-				return nil
-			}
-
-			// This is not for our server, so we can continue
-			hostPort := net.JoinHostPort(dnsBuckets[index].Host, string(dnsBuckets[index].Port))
-			if globalDomainIPs.Intersection(set.CreateStringSet(hostPort)).IsEmpty() {
-				return nil
-			}
-
-			// We go to here, so we know the bucket no longer exists,
-			// but is registered in DNS to this server
-			if err := globalDNSConfig.DeleteRecord(dnsBuckets[index]); err != nil {
-				return fmt.Errorf("Failed to remove DNS entry for %s due to %w",
-					dnsBuckets[index].Key, err)
-			}
-
-			return nil
-		}, index)
+	for _, bucket := range bucketsInConflict.ToSlice() {
+		logger.LogIf(GlobalContext, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", bucket, globalDomainIPs.ToSlice()))
 	}
 
-	for _, err := range g.Wait() {
-		if err != nil {
-			logger.LogIf(context.Background(), err)
+	// Remove buckets that are in DNS for this server, but aren't local
+	for bucket, records := range dnsBuckets {
+		if bucketsSet.Contains(bucket) {
+			continue
+		}
+
+		if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(records)...)).IsEmpty() {
+			// This is not for our server, so we can continue
+			continue
+		}
+
+		// We go to here, so we know the bucket no longer exists,
+		// but is registered in DNS to this server
+		if err = globalDNSConfig.Delete(bucket); err != nil {
+			logger.LogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w",
+				bucket, err))
 		}
 	}
 }
@@ -271,29 +265,24 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	listBuckets := objectAPI.ListBuckets
 
 	accessKey, owner, s3Error := checkRequestAuthTypeToAccessKey(ctx, r, policy.ListAllMyBucketsAction, "", "")
-	if s3Error != ErrNone {
+	if s3Error != ErrNone && s3Error != ErrAccessDenied {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// If etcd, dns federation configured list buckets from etcd.
 	var bucketsInfo []BucketInfo
-	if globalDNSConfig != nil {
+	if globalDNSConfig != nil && globalBucketFederation {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && err != dns.ErrNoEntriesFound {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		bucketSet := set.NewStringSet()
-		for _, dnsRecord := range dnsBuckets {
-			if bucketSet.Contains(dnsRecord.Key) {
-				continue
-			}
+		for _, dnsRecords := range dnsBuckets {
 			bucketsInfo = append(bucketsInfo, BucketInfo{
-				Name:    dnsRecord.Key,
-				Created: dnsRecord.CreationDate,
+				Name:    dnsRecords[0].Key,
+				Created: dnsRecords[0].CreationDate,
 			})
-			bucketSet.Add(dnsRecord.Key)
 		}
 	} else {
 		// Invoke the list buckets.
@@ -305,32 +294,43 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		}
 	}
 
-	// Set prefix value for "s3:prefix" policy conditionals.
-	r.Header.Set("prefix", "")
+	if s3Error == ErrAccessDenied {
+		// Set prefix value for "s3:prefix" policy conditionals.
+		r.Header.Set("prefix", "")
 
-	// Set delimiter value for "s3:delimiter" policy conditionals.
-	r.Header.Set("delimiter", SlashSeparator)
+		// Set delimiter value for "s3:delimiter" policy conditionals.
+		r.Header.Set("delimiter", SlashSeparator)
 
-	// err will be nil here as we already called this function
-	// earlier in this request.
-	claims, _ := getClaimsFromToken(r)
-	var newBucketsInfo []BucketInfo
-	for _, bucketInfo := range bucketsInfo {
-		if globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     accessKey,
-			Action:          iampolicy.ListBucketAction,
-			BucketName:      bucketInfo.Name,
-			ConditionValues: getConditionValues(r, "", accessKey, claims),
-			IsOwner:         owner,
-			ObjectName:      "",
-			Claims:          claims,
-		}) {
-			newBucketsInfo = append(newBucketsInfo, bucketInfo)
+		// err will be nil here as we already called this function
+		// earlier in this request.
+		claims, _ := getClaimsFromToken(r)
+		n := 0
+		// Use the following trick to filter in place
+		// https://github.com/golang/go/wiki/SliceTricks#filter-in-place
+		for _, bucketInfo := range bucketsInfo {
+			if globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     accessKey,
+				Action:          iampolicy.ListBucketAction,
+				BucketName:      bucketInfo.Name,
+				ConditionValues: getConditionValues(r, "", accessKey, claims),
+				IsOwner:         owner,
+				ObjectName:      "",
+				Claims:          claims,
+			}) {
+				bucketsInfo[n] = bucketInfo
+				n++
+			}
+		}
+		bucketsInfo = bucketsInfo[:n]
+		// No buckets can be filtered return access denied error.
+		if len(bucketsInfo) == 0 {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+			return
 		}
 	}
 
 	// Generate response.
-	response := generateListBucketsResponse(newBucketsInfo)
+	response := generateListBucketsResponse(bucketsInfo)
 	encodedSuccessResponse := encodeResponse(response)
 
 	// Write response.
@@ -352,6 +352,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Content-Md5 is requied should be set
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
@@ -359,34 +366,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Content-Md5 is requied should be set
-	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Allocate incoming content length bytes.
-	var deleteXMLBytes []byte
-	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
-	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
-		deleteXMLBytes = make([]byte, maxBodySize)
-	} else {
-		deleteXMLBytes = make([]byte, r.ContentLength)
-	}
-
-	// Read incoming body XML bytes.
-	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
-		logger.LogIf(ctx, err, logger.Application)
-		writeErrorResponse(ctx, w, toAdminAPIErr(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
+	// The max. XML contains 100000 object names (each at most 1024 bytes long) + XML overhead
+	const maxBodySize = 2 * 100000 * 1024
 
 	// Unmarshal list of keys to be deleted.
 	deleteObjects := &DeleteObjectsRequest{}
-	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
+	if err := xmlDecoder(r.Body, deleteObjects, maxBodySize); err != nil {
 		logger.LogIf(ctx, err, logger.Application)
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -411,11 +398,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			}
 			continue
 		}
-		govBypassPerms := checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName)
-		if _, err := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn, govBypassPerms); err != ErrNone {
-			dErrs[index] = err
-			continue
+
+		if _, ok := globalBucketObjectLockConfig.Get(bucket); ok {
+			if err := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn); err != ErrNone {
+				dErrs[index] = err
+				continue
+			}
 		}
+
 		// Avoid duplicate objects, we use map to filter them out.
 		if _, ok := objectsToDelete[object.ObjectName]; !ok {
 			objectsToDelete[object.ObjectName] = index
@@ -533,7 +523,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	if globalDNSConfig != nil {
-		if _, err := globalDNSConfig.Get(bucket); err != nil {
+		sr, err := globalDNSConfig.Get(bucket)
+		if err != nil {
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
@@ -550,7 +541,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				if err = globalDNSConfig.Put(bucket); err != nil {
-					objectAPI.DeleteBucket(ctx, bucket)
+					objectAPI.DeleteBucket(ctx, bucket, false)
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
@@ -566,7 +557,15 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 			return
 
 		}
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBucketAlreadyOwnedByYou), r.URL, guessIsBrowserReq(r))
+		apiErr := ErrBucketAlreadyExists
+		if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
+			apiErr = ErrBucketAlreadyOwnedByYou
+		}
+		// No IPs seem to intersect, this means that bucket exists but has
+		// different IP addresses perhaps from a different deployment.
+		// bucket names are globally unique in federation at a given
+		// path prefix, name collision is not allowed. Return appropriate error.
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -577,14 +576,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if objectLockEnabled {
+	if objectLockEnabled && !globalIsGateway {
 		configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
 		if err = saveConfig(ctx, objectAPI, configFile, []byte(bucketObjectLockEnabledConfig)); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		globalBucketObjectLockConfig.Set(bucket, Retention{})
-		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, Retention{})
+		globalBucketObjectLockConfig.Set(bucket, objectlock.Retention{})
+		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, objectlock.Retention{})
 	}
 
 	// Make sure to add Location information here only for bucket
@@ -756,10 +755,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 	rawReader := hashReader
 	pReader := NewPutObjReader(rawReader, nil, nil)
-	var objectEncryptionKey []byte
+	var objectEncryptionKey crypto.ObjectKey
 
+	// Check if bucket encryption is enabled
+	_, encEnabled := globalBucketSSEConfigSys.Get(bucket)
 	// This request header needs to be set prior to setting ObjectOptions
-	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
+	if (globalAutoEncryption || encEnabled) && !crypto.SSEC.IsRequested(r.Header) {
 		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 	}
 	// get gateway encryption options
@@ -796,7 +797,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
-			pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
+			pReader = NewPutObjReader(rawReader, hashReader, &objectEncryptionKey)
 		}
 	}
 
@@ -895,15 +896,39 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+	forceDelete := false
+	if value := r.Header.Get(xhttp.MinIOForceDelete); value != "" {
+		switch value {
+		case "true":
+			forceDelete = true
+		case "false":
+		default:
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
+	if forceDelete {
+		if s3Error := checkRequestAuthType(ctx, r, policy.ForceDeleteBucketAction, bucket, ""); s3Error != ErrNone {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	} else {
+		if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
+	if _, ok := globalBucketObjectLockConfig.Get(bucket); (ok || globalWORMEnabled) && forceDelete {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	deleteBucket := objectAPI.DeleteBucket
 
 	// Attempt to delete bucket.
-	if err := deleteBucket(ctx, bucket); err != nil {
+	if err := deleteBucket(ctx, bucket, forceDelete); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1005,7 +1030,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	config, err := parseObjectLockConfig(r.Body)
+	config, err := objectlock.ParseObjectLockConfig(r.Body)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		apiErr.Description = err.Error()
@@ -1017,7 +1042,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 	if err != nil {
 		aerr := toAPIError(ctx, err)
 		if err == errConfigNotFound {
-			aerr = errorCodes.ToAPIErr(ErrMethodNotAllowed)
+			aerr = errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed)
 		}
 		writeErrorResponse(ctx, w, aerr, r.URL, guessIsBrowserReq(r))
 		return
@@ -1099,7 +1124,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 			return
 		}
 
-		if configData, err = xml.Marshal(newObjectLockConfig()); err != nil {
+		if configData, err = xml.Marshal(objectlock.NewObjectLockConfig()); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}

cmd/bucket-handlers_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"encoding/xml"
 	"io/ioutil"
 	"net/http"
@@ -29,6 +28,51 @@ import (
 	"github.com/minio/minio/pkg/auth"
 )
 
+// Wrapper for calling RemoveBucket HTTP handler tests for both XL multiple disks and single node setup.
+func TestRemoveBucketHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testRemoveBucketHandler, []string{"RemoveBucket"})
+}
+
+func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials auth.Credentials, t *testing.T) {
+	_, err := obj.PutObject(GlobalContext, bucketName, "test-object", mustGetPutObjReader(t, bytes.NewBuffer([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{})
+	// if object upload fails stop the test.
+	if err != nil {
+		t.Fatalf("Error uploading object: <ERROR> %v", err)
+	}
+
+	// initialize httptest Recorder, this records any mutations to response writer inside the handler.
+	rec := httptest.NewRecorder()
+	// construct HTTP request for DELETE bucket.
+	req, err := newTestSignedRequestV4("DELETE", getBucketLocationURL("", bucketName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil)
+	if err != nil {
+		t.Fatalf("Test %s: Failed to create HTTP request for RemoveBucketHandler: <ERROR> %v", instanceType, err)
+	}
+	// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+	// Call the ServeHTTP to execute the handler.
+	apiRouter.ServeHTTP(rec, req)
+	switch rec.Code {
+	case http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent:
+		t.Fatalf("Test %v: expected failure, but succeeded with %v", instanceType, rec.Code)
+	}
+
+	// Verify response of the V2 signed HTTP request.
+	// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+	recV2 := httptest.NewRecorder()
+	// construct HTTP request for DELETE bucket.
+	reqV2, err := newTestSignedRequestV2("DELETE", getBucketLocationURL("", bucketName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil)
+	if err != nil {
+		t.Fatalf("Test %s: Failed to create HTTP request for RemoveBucketHandler: <ERROR> %v", instanceType, err)
+	}
+	// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+	// Call the ServeHTTP to execute the handler.
+	apiRouter.ServeHTTP(recV2, reqV2)
+	switch recV2.Code {
+	case http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent:
+		t.Fatalf("Test %v: expected failure, but succeeded with %v", instanceType, recV2.Code)
+	}
+}
+
 // Wrapper for calling GetBucketPolicy HTTP handler tests for both XL multiple disks and single node setup.
 func TestGetBucketLocationHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testGetBucketLocationHandler, []string{"GetBucketLocation"})
@@ -625,7 +669,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
+		_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)

cmd/bucket-lifecycle-handler.go

@@ -22,9 +22,10 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/lifecycle"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 const (
@@ -48,6 +49,12 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
+	// PutBucketLifecycle always needs a Content-Md5
+	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
@@ -59,15 +66,9 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// PutBucketLifecycle always needs a Content-Md5
-	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
 	bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/bucket-lifecycle-handlers_test.go

@@ -0,0 +1,303 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/xml"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/minio/minio/pkg/auth"
+)
+
+// Test S3 Bucket lifecycle APIs with wrong credentials
+func TestBucketLifecycleWrongCredentials(t *testing.T) {
+	ExecObjectLayerAPITest(t, testBucketLifecycleHandlersWrongCredentials, []string{"GetBucketLifecycle", "PutBucketLifecycle", "DeleteBucketLifecycle"})
+}
+
+// Test for authentication
+func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials auth.Credentials, t *testing.T) {
+	// test cases with sample input and expected output.
+	testCases := []struct {
+		method     string
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// Sent body
+		body []byte
+		// Expected response
+		expectedRespStatus int
+		lifecycleResponse  []byte
+		errorResponse      APIErrorResponse
+		shouldPass         bool
+	}{
+		// GET empty credentials
+		{
+			method: "GET", bucketName: bucketName,
+			accessKey:          "",
+			secretKey:          "",
+			expectedRespStatus: http.StatusForbidden,
+			lifecycleResponse:  []byte(""),
+			errorResponse: APIErrorResponse{
+				Resource: SlashSeparator + bucketName + SlashSeparator,
+				Code:     "AccessDenied",
+				Message:  "Access Denied.",
+			},
+			shouldPass: false,
+		},
+		// GET wrong credentials
+		{
+			method: "GET", bucketName: bucketName,
+			accessKey:          "abcd",
+			secretKey:          "abcd",
+			expectedRespStatus: http.StatusForbidden,
+			lifecycleResponse:  []byte(""),
+			errorResponse: APIErrorResponse{
+				Resource: SlashSeparator + bucketName + SlashSeparator,
+				Code:     "InvalidAccessKeyId",
+				Message:  "The access key ID you provided does not exist in our records.",
+			},
+			shouldPass: false,
+		},
+		// PUT empty credentials
+		{
+			method:             "PUT",
+			bucketName:         bucketName,
+			accessKey:          "",
+			secretKey:          "",
+			expectedRespStatus: http.StatusForbidden,
+			lifecycleResponse:  []byte(""),
+			errorResponse: APIErrorResponse{
+				Resource: SlashSeparator + bucketName + SlashSeparator,
+				Code:     "AccessDenied",
+				Message:  "Access Denied.",
+			},
+			shouldPass: false,
+		},
+		// PUT wrong credentials
+		{
+			method:             "PUT",
+			bucketName:         bucketName,
+			accessKey:          "abcd",
+			secretKey:          "abcd",
+			expectedRespStatus: http.StatusForbidden,
+			lifecycleResponse:  []byte(""),
+			errorResponse: APIErrorResponse{
+				Resource: SlashSeparator + bucketName + SlashSeparator,
+				Code:     "InvalidAccessKeyId",
+				Message:  "The access key ID you provided does not exist in our records.",
+			},
+			shouldPass: false,
+		},
+		// DELETE empty credentials
+		{
+			method:             "DELETE",
+			bucketName:         bucketName,
+			accessKey:          "",
+			secretKey:          "",
+			expectedRespStatus: http.StatusForbidden,
+			lifecycleResponse:  []byte(""),
+			errorResponse: APIErrorResponse{
+				Resource: SlashSeparator + bucketName + SlashSeparator,
+				Code:     "AccessDenied",
+				Message:  "Access Denied.",
+			},
+			shouldPass: false,
+		},
+		// DELETE wrong credentials
+		{
+			method:             "DELETE",
+			bucketName:         bucketName,
+			accessKey:          "abcd",
+			secretKey:          "abcd",
+			expectedRespStatus: http.StatusForbidden,
+			lifecycleResponse:  []byte(""),
+			errorResponse: APIErrorResponse{
+				Resource: SlashSeparator + bucketName + SlashSeparator,
+				Code:     "InvalidAccessKeyId",
+				Message:  "The access key ID you provided does not exist in our records.",
+			},
+			shouldPass: false,
+		},
+	}
+
+	testBucketLifecycle(obj, instanceType, bucketName, apiRouter, t, testCases)
+}
+
+// Test S3 Bucket lifecycle APIs
+func TestBucketLifecycle(t *testing.T) {
+	ExecObjectLayerAPITest(t, testBucketLifecycleHandlers, []string{"GetBucketLifecycle", "PutBucketLifecycle", "DeleteBucketLifecycle"})
+}
+
+// Simple tests of bucket lifecycle: PUT, GET, DELETE.
+// Tests are related and the order is important.
+func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	creds auth.Credentials, t *testing.T) {
+
+	// test cases with sample input and expected output.
+	testCases := []struct {
+		method     string
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// Sent body
+		body []byte
+		// Expected response
+		expectedRespStatus int
+		lifecycleResponse  []byte
+		errorResponse      APIErrorResponse
+		shouldPass         bool
+	}{
+		// Test case - 1.
+		// Filter contains more than (Prefix,Tag,And) rule
+		{
+			method:             "PUT",
+			bucketName:         bucketName,
+			accessKey:          creds.AccessKey,
+			secretKey:          creds.SecretKey,
+			body:               []byte(`<LifecycleConfiguration><Rule><ID>id</ID><Filter><Prefix>logs/</Prefix><Tag><Key>Key1</Key><Value>Value1</Value></Tag></Filter><Status>Enabled</Status><Expiration><Days>365</Days></Expiration></Rule></LifecycleConfiguration>`),
+			expectedRespStatus: http.StatusBadRequest,
+			lifecycleResponse:  []byte(``),
+			errorResponse: APIErrorResponse{
+				Resource: SlashSeparator + bucketName + SlashSeparator,
+				Code:     "InvalidRequest",
+				Message:  "Filter must have exactly one of Prefix, Tag, or And specified",
+			},
+
+			shouldPass: false,
+		},
+		// Date contains wrong format
+		{
+			method:             "PUT",
+			bucketName:         bucketName,
+			accessKey:          creds.AccessKey,
+			secretKey:          creds.SecretKey,
+			body:               []byte(`<LifecycleConfiguration><Rule><ID>id</ID><Filter><Prefix>logs/</Prefix><Tag><Key>Key1</Key><Value>Value1</Value></Tag></Filter><Status>Enabled</Status><Expiration><Date>365</Date></Expiration></Rule></LifecycleConfiguration>`),
+			expectedRespStatus: http.StatusBadRequest,
+			lifecycleResponse:  []byte(``),
+			errorResponse: APIErrorResponse{
+				Resource: SlashSeparator + bucketName + SlashSeparator,
+				Code:     "InvalidRequest",
+				Message:  "Date must be provided in ISO 8601 format",
+			},
+
+			shouldPass: false,
+		},
+		{
+			method:             "PUT",
+			bucketName:         bucketName,
+			accessKey:          creds.AccessKey,
+			secretKey:          creds.SecretKey,
+			body:               []byte(`<?xml version="1.0" encoding="UTF-8"?><LifecycleConfiguration><Rule><ID>id</ID><Filter><Prefix>logs/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>365</Days></Expiration></Rule></LifecycleConfiguration>`),
+			expectedRespStatus: http.StatusOK,
+			lifecycleResponse:  []byte(``),
+			errorResponse:      APIErrorResponse{},
+			shouldPass:         true,
+		},
+		{
+			method:             "GET",
+			accessKey:          creds.AccessKey,
+			secretKey:          creds.SecretKey,
+			bucketName:         bucketName,
+			body:               []byte(``),
+			expectedRespStatus: http.StatusOK,
+			lifecycleResponse:  []byte(`<LifecycleConfiguration><Rule><ID>id</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><Expiration><Days>365</Days></Expiration></Rule></LifecycleConfiguration>`),
+			errorResponse:      APIErrorResponse{},
+			shouldPass:         true,
+		},
+		{
+			method:             "DELETE",
+			accessKey:          creds.AccessKey,
+			secretKey:          creds.SecretKey,
+			bucketName:         bucketName,
+			body:               []byte(``),
+			expectedRespStatus: http.StatusNoContent,
+			lifecycleResponse:  []byte(``),
+			errorResponse:      APIErrorResponse{},
+			shouldPass:         true,
+		},
+		{
+			method:             "GET",
+			accessKey:          creds.AccessKey,
+			secretKey:          creds.SecretKey,
+			bucketName:         bucketName,
+			body:               []byte(``),
+			expectedRespStatus: http.StatusNotFound,
+			lifecycleResponse:  []byte(``),
+			errorResponse: APIErrorResponse{
+				Resource: SlashSeparator + bucketName + SlashSeparator,
+				Code:     "NoSuchLifecycleConfiguration",
+				Message:  "The lifecycle configuration does not exist",
+			},
+			shouldPass: false,
+		},
+	}
+
+	testBucketLifecycle(obj, instanceType, bucketName, apiRouter, t, testCases)
+}
+
+// testBucketLifecycle is a generic testing of lifecycle requests
+func testBucketLifecycle(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	t *testing.T, testCases []struct {
+		method             string
+		bucketName         string
+		accessKey          string
+		secretKey          string
+		body               []byte
+		expectedRespStatus int
+		lifecycleResponse  []byte
+		errorResponse      APIErrorResponse
+		shouldPass         bool
+	}) {
+
+	for i, testCase := range testCases {
+		// initialize httptest Recorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request
+		req, err := newTestSignedRequestV4(testCase.method, getBucketLifecycleURL("", testCase.bucketName),
+			int64(len(testCase.body)), bytes.NewReader(testCase.body), testCase.accessKey, testCase.secretKey, nil)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+		}
+		if testCase.shouldPass && !bytes.Equal(testCase.lifecycleResponse, rec.Body.Bytes()) {
+			t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.lifecycleResponse), rec.Body.String())
+		}
+		errorResponse := APIErrorResponse{}
+		err = xml.Unmarshal(rec.Body.Bytes(), &errorResponse)
+		if err != nil && !testCase.shouldPass {
+			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, rec.Body.String())
+		}
+		if errorResponse.Resource != testCase.errorResponse.Resource {
+			t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource)
+		}
+		if errorResponse.Message != testCase.errorResponse.Message {
+			t.Errorf("Test %d: %s: Expected the error message to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Message, errorResponse.Message)
+		}
+		if errorResponse.Code != testCase.errorResponse.Code {
+			t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code)
+		}
+	}
+}

cmd/bucket-notification-handlers.go

@@ -30,8 +30,8 @@ import (
 	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/policy"
 )
 
 const (
@@ -84,8 +84,8 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
 		config.SetRegion(globalServerRegion)
+		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
 		notificationBytes, err := xml.Marshal(config)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/bucket-policy-handlers.go

@@ -24,7 +24,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 const (
@@ -77,7 +77,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/bucket-policy-handlers_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -29,8 +28,8 @@ import (
 	"testing"
 
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/policy"
-	"github.com/minio/minio/pkg/policy/condition"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/policy/condition"
 )
 
 func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {
@@ -103,7 +102,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T) {
 
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName1, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, ""); err != nil {
 		t.Fatal(err)
 	}
 

cmd/common-main.go

@@ -18,8 +18,10 @@ package cmd
 
 import (
 	"crypto/x509"
+	"encoding/gob"
 	"errors"
 	"net"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -34,6 +36,17 @@ import (
 	"github.com/minio/minio/pkg/env"
 )
 
+func init() {
+	logger.Init(GOPATH, GOROOT)
+	logger.RegisterError(config.FmtError)
+
+	// Initialize globalConsoleSys system
+	globalConsoleSys = NewConsoleLogger(GlobalContext)
+	logger.AddTarget(globalConsoleSys)
+
+	gob.Register(StorageErr(""))
+}
+
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
 	if (globalAutoEncryption || GlobalKMS != nil) && !objAPI.IsEncryptionSupported() {
 		logger.Fatal(errInvalidArgument,
@@ -213,10 +226,22 @@ func handleCommonEnvVars() {
 		globalConfigEncrypted = true
 	}
 
+	if env.IsSet(config.EnvAccessKeyOld) && env.IsSet(config.EnvSecretKeyOld) {
+		oldCred, err := auth.CreateCredentials(env.Get(config.EnvAccessKeyOld, ""), env.Get(config.EnvSecretKeyOld, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate the old credentials inherited from the shell environment")
+		}
+		globalOldCred = oldCred
+		os.Unsetenv(config.EnvAccessKeyOld)
+		os.Unsetenv(config.EnvSecretKeyOld)
+	}
+
 	globalWORMEnabled, err = config.LookupWorm()
 	if err != nil {
 		logger.Fatal(config.ErrInvalidWormValue(err), "Invalid worm configuration")
 	}
+
 }
 
 func logStartupMessage(msg string) {

cmd/config-current.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"strings"
 	"sync"
@@ -26,7 +25,6 @@ import (
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
 	"github.com/minio/minio/cmd/config/etcd"
-	xetcd "github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/config/etcd/dns"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
@@ -38,6 +36,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 func initHelp() {
@@ -251,7 +250,7 @@ func validateConfig(s config.Config) error {
 		}
 	}
 	{
-		kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewCustomHTTPTransport())
+		kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
 		if err != nil {
 			return err
 		}
@@ -269,17 +268,27 @@ func validateConfig(s config.Config) error {
 	}
 
 	if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
-		NewCustomHTTPTransport(), xhttp.DrainBody); err != nil {
+		NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
 		return err
 	}
 
-	if _, err := xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
-		globalRootCAs); err != nil {
-		return err
+	{
+		cfg, err := xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
+			globalRootCAs)
+		if err != nil {
+			return err
+		}
+		if cfg.Enabled {
+			conn, cerr := cfg.Connect()
+			if cerr != nil {
+				return cerr
+			}
+			conn.Close()
+		}
 	}
 
 	if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
-		NewCustomHTTPTransport(), xhttp.DrainBody); err != nil {
+		NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
 		return err
 	}
 
@@ -287,12 +296,12 @@ func validateConfig(s config.Config) error {
 		return err
 	}
 
-	return notify.TestNotificationTargets(s, GlobalServiceDoneCh, NewCustomHTTPTransport(),
+	return notify.TestNotificationTargets(s, GlobalServiceDoneCh, NewGatewayHTTPTransport(),
 		globalNotificationSys.ConfiguredTargetIDs())
 }
 
 func lookupConfigs(s config.Config) {
-	ctx := context.Background()
+	ctx := GlobalContext
 
 	var err error
 	if !globalActiveCred.IsValid() {
@@ -303,18 +312,25 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
-	etcdCfg, err := xetcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
+	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 	}
 
 	if etcdCfg.Enabled {
-		globalEtcdClient, err = xetcd.New(etcdCfg)
+		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 		}
 	}
 
+	// Bucket federation is 'true' only when IAM assets are not namespaced
+	// per tenant and all tenants interested in globally available users
+	// if namespace was requested such as specifying etcdPathPrefix then
+	// we assume that users are interested in global bucket support
+	// but not federation.
+	globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
+
 	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
 		globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
 			dns.DomainNames(globalDomainNames),
@@ -355,7 +371,7 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
-	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewCustomHTTPTransport())
+	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to setup KMS config: %w", err))
 	}
@@ -374,13 +390,13 @@ func lookupConfigs(s config.Config) {
 	}
 
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
-		NewCustomHTTPTransport(), xhttp.DrainBody)
+		NewGatewayHTTPTransport(), xhttp.DrainBody)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
 	}
 
 	opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
-		NewCustomHTTPTransport(), xhttp.DrainBody)
+		NewGatewayHTTPTransport(), xhttp.DrainBody)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OPA: %w", err))
 	}
@@ -405,18 +421,37 @@ func lookupConfigs(s config.Config) {
 	for _, l := range loggerCfg.HTTP {
 		if l.Enabled {
 			// Enable http logging
-			logger.AddTarget(http.New(l.Endpoint, loggerUserAgent, string(logger.All), NewCustomHTTPTransport()))
+			logger.AddTarget(
+				http.New(http.WithEndpoint(l.Endpoint),
+					http.WithAuthToken(l.AuthToken),
+					http.WithUserAgent(loggerUserAgent),
+					http.WithLogKind(string(logger.All)),
+					http.WithTransport(NewGatewayHTTPTransport()),
+				),
+			)
 		}
 	}
 
 	for _, l := range loggerCfg.Audit {
 		if l.Enabled {
 			// Enable http audit logging
-			logger.AddAuditTarget(http.New(l.Endpoint, loggerUserAgent, string(logger.All), NewCustomHTTPTransport()))
+			logger.AddAuditTarget(
+				http.New(http.WithEndpoint(l.Endpoint),
+					http.WithAuthToken(l.AuthToken),
+					http.WithUserAgent(loggerUserAgent),
+					http.WithLogKind(string(logger.All)),
+					http.WithTransport(NewGatewayHTTPTransport()),
+				),
+			)
 		}
 	}
 
-	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalServiceDoneCh, NewCustomHTTPTransport())
+	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalServiceDoneCh, NewGatewayHTTPTransport())
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
+	}
+
+	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalServiceDoneCh, NewGatewayHTTPTransport())
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
@@ -462,6 +497,20 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 
 	envHelp := config.HelpKVS{}
 	if envOnly {
+		// Only for multiple targets, make sure
+		// to list the ENV, for regular k/v EnableKey is
+		// implicit, for ENVs we cannot make it implicit.
+		if subSysHelp.MultipleTargets {
+			envK := config.EnvPrefix + strings.Join([]string{
+				strings.ToTitle(subSys), strings.ToTitle(madmin.EnableKey),
+			}, config.EnvWordDelimiter)
+			envHelp = append(envHelp, config.HelpKV{
+				Key:         envK,
+				Description: fmt.Sprintf("enable %s target, default is 'off'", subSys),
+				Optional:    false,
+				Type:        "on|off",
+			})
+		}
 		for _, hkv := range h {
 			envK := config.EnvPrefix + strings.Join([]string{
 				strings.ToTitle(subSys), strings.ToTitle(hkv.Key),
@@ -503,11 +552,11 @@ func newSrvConfig(objAPI ObjectLayer) error {
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return saveServerConfig(context.Background(), objAPI, globalServerConfig)
+	return saveServerConfig(GlobalContext, objAPI, globalServerConfig)
 }
 
 func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
-	return readServerConfig(context.Background(), objAPI)
+	return readServerConfig(GlobalContext, objAPI)
 }
 
 // loadConfig - loads a new config from disk, overrides params

cmd/config-encrypted.go

@@ -21,15 +21,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
-	"strings"
 	"unicode/utf8"
 
 	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -50,12 +47,18 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 	//    of the object layer.
 	retryTimerCh := newRetryTimerSimple(doneCh)
 	var stop bool
+
+	rquorum := InsufficientReadQuorum{}
+	wquorum := InsufficientWriteQuorum{}
+	bucketNotFound := BucketNotFound{}
+
 	for !stop {
 		select {
 		case <-retryTimerCh:
 			if encrypted, err = checkBackendEncrypted(objAPI); err != nil {
-				if err == errDiskNotFound ||
-					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) {
+				if errors.Is(err, errDiskNotFound) ||
+					errors.As(err, &rquorum) ||
+					errors.As(err, &bucketNotFound) {
 					logger.Info("Waiting for config backend to be encrypted..")
 					continue
 				}
@@ -84,15 +87,10 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 			return nil
 		}
 		if !globalActiveCred.IsValid() {
-			return errInvalidArgument
+			return config.ErrMissingCredentialsBackendEncrypted(nil)
 		}
 	}
 
-	activeCredOld, err := getOldCreds()
-	if err != nil {
-		return err
-	}
-
 	doneCh = make(chan struct{})
 	defer close(doneCh)
 
@@ -106,10 +104,11 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		select {
 		case <-retryTimerCh:
 			// Migrate IAM configuration
-			if err = migrateConfigPrefixToEncrypted(objAPI, activeCredOld, encrypted); err != nil {
-				if err == errDiskNotFound ||
-					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+			if err = migrateConfigPrefixToEncrypted(objAPI, globalOldCred, encrypted); err != nil {
+				if errors.Is(err, errDiskNotFound) ||
+					errors.As(err, &rquorum) ||
+					errors.As(err, &wquorum) ||
+					errors.As(err, &bucketNotFound) {
 					logger.Info("Waiting for config backend to be encrypted..")
 					continue
 				}
@@ -140,7 +139,7 @@ func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool,
 }
 
 func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
-	data, err := readConfig(context.Background(), objAPI, backendEncryptedFile)
+	data, err := readConfig(GlobalContext, objAPI, backendEncryptedFile)
 	if err != nil && err != errConfigNotFound {
 		return false, err
 	}
@@ -164,23 +163,8 @@ func decryptData(edata []byte, creds ...auth.Credentials) ([]byte, error) {
 	return data, err
 }
 
-func getOldCreds() (activeCredOld auth.Credentials, err error) {
-	accessKeyOld := env.Get(config.EnvAccessKeyOld, "")
-	secretKeyOld := env.Get(config.EnvSecretKeyOld, "")
-	if accessKeyOld != "" && secretKeyOld != "" {
-		activeCredOld, err = auth.CreateCredentials(accessKeyOld, secretKeyOld)
-		if err != nil {
-			return activeCredOld, err
-		}
-		// Once we have obtained the rotating creds
-		os.Unsetenv(config.EnvAccessKeyOld)
-		os.Unsetenv(config.EnvSecretKeyOld)
-	}
-	return activeCredOld, nil
-}
-
-func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client) error {
+	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
 
 	encrypted, err := checkBackendEtcdEncrypted(ctx, client)
@@ -206,20 +190,15 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		}
 	}
 
-	activeCredOld, err := getOldCreds()
-	if err != nil {
-		return err
-	}
-
 	if encrypted {
 		// No key rotation requested, and backend is
 		// already encrypted. We proceed without migration.
-		if !activeCredOld.IsValid() {
+		if !globalOldCred.IsValid() {
 			return nil
 		}
 
 		// No real reason to rotate if old and new creds are same.
-		if activeCredOld.Equal(globalActiveCred) {
+		if globalOldCred.Equal(globalActiveCred) {
 			return nil
 		}
 
@@ -254,8 +233,8 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 
 		var data []byte
 		// Is rotating of creds requested?
-		if activeCredOld.IsValid() {
-			data, err = decryptData(cdata, activeCredOld, globalActiveCred)
+		if globalOldCred.IsValid() {
+			data, err = decryptData(cdata, globalOldCred, globalActiveCred)
 			if err != nil {
 				if err == madmin.ErrMaliciousData {
 					return config.ErrInvalidRotatingCredentialsBackendEncrypted(nil)
@@ -267,7 +246,12 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		}
 
 		if !utf8.Valid(data) {
-			return errors.New("config data not in plain-text form")
+			_, err = decryptData(data, globalActiveCred)
+			if err == nil {
+				// Config is already encrypted with right keys
+				continue
+			}
+			return errors.New("config data not in plain-text form or encrypted")
 		}
 
 		cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
@@ -279,9 +263,11 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 			return err
 		}
 	}
-	if encrypted && globalActiveCred.IsValid() {
+
+	if encrypted && globalActiveCred.IsValid() && globalOldCred.IsValid() {
 		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
 	}
+
 	return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationComplete)
 }
 
@@ -302,14 +288,15 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 		logger.Info("Attempting encryption of all config, IAM users and policies on MinIO backend")
 	}
 
-	err := saveConfig(context.Background(), objAPI, backendEncryptedFile, backendEncryptedMigrationIncomplete)
+	err := saveConfig(GlobalContext, objAPI, backendEncryptedFile, backendEncryptedMigrationIncomplete)
 	if err != nil {
 		return err
 	}
 
 	var marker string
 	for {
-		res, err := objAPI.ListObjects(context.Background(), minioMetaBucket, minioConfigPrefix, marker, "", maxObjectList)
+		res, err := objAPI.ListObjects(GlobalContext, minioMetaBucket,
+			minioConfigPrefix, marker, "", maxObjectList)
 		if err != nil {
 			return err
 		}
@@ -319,7 +306,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 				cencdata []byte
 			)
 
-			cdata, err = readConfig(context.Background(), objAPI, obj.Name)
+			cdata, err = readConfig(GlobalContext, objAPI, obj.Name)
 			if err != nil {
 				return err
 			}
@@ -339,7 +326,12 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 			}
 
 			if !utf8.Valid(data) {
-				return errors.New("config data not in plain-text form")
+				_, err = decryptData(data, globalActiveCred)
+				if err == nil {
+					// Config is already encrypted with right keys
+					continue
+				}
+				return errors.New("config data not in plain-text form or encrypted")
 			}
 
 			cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
@@ -347,7 +339,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 				return err
 			}
 
-			if err = saveConfig(context.Background(), objAPI, obj.Name, cencdata); err != nil {
+			if err = saveConfig(GlobalContext, objAPI, obj.Name, cencdata); err != nil {
 				return err
 			}
 		}
@@ -359,9 +351,9 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 		marker = res.NextMarker
 	}
 
-	if encrypted && globalActiveCred.IsValid() {
+	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
 		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
 	}
 
-	return saveConfig(context.Background(), objAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
+	return saveConfig(GlobalContext, objAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 }

cmd/config-migrate.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -2430,7 +2429,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	defer func() {
 		if err == nil {
 			if globalEtcdClient != nil {
-				deleteKeyEtcd(context.Background(), globalEtcdClient, configFile)
+				deleteKeyEtcd(GlobalContext, globalEtcdClient, configFile)
 			} else {
 				// Rename config.json to config.json.deprecated only upon
 				// success of this function.
@@ -2440,7 +2439,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	}()
 
 	// Verify if backend already has the file (after holding lock)
-	if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
+	if err = checkConfig(GlobalContext, objAPI, configFile); err != errConfigNotFound {
 		return err
 	} // if errConfigNotFound proceed to migrate..
 
@@ -2466,7 +2465,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		// Initialize the server config, if no config exists.
 		return newSrvConfig(objAPI)
 	}
-	return saveServerConfig(context.Background(), objAPI, config)
+	return saveServerConfig(GlobalContext, objAPI, config)
 }
 
 // Migrates '.minio.sys/config.json' to v33.
@@ -2502,7 +2501,7 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
 }
 
 func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
-	data, err := readConfig(context.Background(), objAPI, configFile)
+	data, err := readConfig(GlobalContext, objAPI, configFile)
 	if err != nil {
 		return false, nil, err
 	}
@@ -2548,7 +2547,7 @@ func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
 	cfg.Version = "28"
 	cfg.KMS = crypto.KMSConfig{}
 
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %w", err)
 	}
 
@@ -2575,7 +2574,7 @@ func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
 	}
 
 	cfg.Version = "29"
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %w", err)
 	}
 
@@ -2607,7 +2606,7 @@ func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
 	cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
 	cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
 
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %w", err)
 	}
 
@@ -2642,7 +2641,7 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
 		AuthToken: "",
 	}
 
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %w", err)
 	}
 
@@ -2672,7 +2671,7 @@ func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
 	cfg.Notify.NSQ = make(map[string]target.NSQArgs)
 	cfg.Notify.NSQ["1"] = target.NSQArgs{}
 
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %w", err)
 	}
 
@@ -2700,7 +2699,7 @@ func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
 
 	cfg.Version = "33"
 
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from '32' to '33' . %w", err)
 	}
 
@@ -2777,7 +2776,7 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
 		notify.SetNotifyWebhook(newCfg, k, args)
 	}
 
-	if err = saveServerConfig(context.Background(), objAPI, newCfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, newCfg); err != nil {
 		return err
 	}
 

cmd/config.go

@@ -20,14 +20,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"fmt"
 	"path"
 	"sort"
 	"strings"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -170,6 +169,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 	}
 
 	var config = config.New()
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.Unmarshal(configData, &config); err != nil {
 		return nil, err
 	}
@@ -210,33 +210,7 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
-	// Initializing configuration needs a retry mechanism for
-	// the following reasons:
-	//  - Read quorum is lost just after the initialization
-	//    of the object layer.
-	//  - Write quorum not met when upgrading configuration
-	//    version is needed.
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	for {
-		select {
-		case <-retryTimerCh:
-			if err := initConfig(objAPI); err != nil {
-				if err == errDiskNotFound ||
-					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-					logger.Info("Waiting for configuration to be initialized..")
-					continue
-				}
-				return err
-			}
-			return nil
-		case <-globalOSSignalCh:
-			return fmt.Errorf("Initializing config sub-system gracefully stopped")
-		}
-	}
+	return initConfig(objAPI)
 }
 
 // NewConfigSys - creates new config system object.

cmd/config/cache/config.go

@@ -28,12 +28,15 @@ import (
 
 // Config represents cache config settings
 type Config struct {
-	Enabled bool     `json:"-"`
-	Drives  []string `json:"drives"`
-	Expiry  int      `json:"expiry"`
-	MaxUse  int      `json:"maxuse"`
-	Quota   int      `json:"quota"`
-	Exclude []string `json:"exclude"`
+	Enabled       bool     `json:"-"`
+	Drives        []string `json:"drives"`
+	Expiry        int      `json:"expiry"`
+	MaxUse        int      `json:"maxuse"`
+	Quota         int      `json:"quota"`
+	Exclude       []string `json:"exclude"`
+	After         int      `json:"after"`
+	WatermarkLow  int      `json:"watermark_low"`
+	WatermarkHigh int      `json:"watermark_high"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
@@ -60,7 +63,18 @@ func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
 	if _cfg.Quota < 0 {
 		return errors.New("config quota value should not be null or negative")
 	}
-
+	if _cfg.After < 0 {
+		return errors.New("cache after value should not be less than 0")
+	}
+	if _cfg.WatermarkLow < 0 || _cfg.WatermarkLow > 100 {
+		return errors.New("config low watermark value should be between 0 and 100")
+	}
+	if _cfg.WatermarkHigh < 0 || _cfg.WatermarkHigh > 100 {
+		return errors.New("config high watermark value should be between 0 and 100")
+	}
+	if _cfg.WatermarkLow > 0 && (_cfg.WatermarkLow >= _cfg.WatermarkHigh) {
+		return errors.New("config low watermark value should be less than high watermark")
+	}
 	return nil
 }
 

cmd/config/cache/help.go

@@ -50,5 +50,23 @@ var (
 			Optional:    true,
 			Type:        "sentence",
 		},
+		config.HelpKV{
+			Key:         After,
+			Description: `minimum accesses before caching an object`,
+			Optional:    true,
+			Type:        "number",
+		},
+		config.HelpKV{
+			Key:         WatermarkLow,
+			Description: `% of cache use at which to stop cache eviction`,
+			Optional:    true,
+			Type:        "number",
+		},
+		config.HelpKV{
+			Key:         WatermarkHigh,
+			Description: `% of cache use at which to start cache eviction`,
+			Optional:    true,
+			Type:        "number",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -26,21 +26,31 @@ import (
 
 // Cache ENVs
 const (
-	Drives  = "drives"
-	Exclude = "exclude"
-	Expiry  = "expiry"
-	MaxUse  = "maxuse"
-	Quota   = "quota"
+	Drives        = "drives"
+	Exclude       = "exclude"
+	Expiry        = "expiry"
+	MaxUse        = "maxuse"
+	Quota         = "quota"
+	After         = "after"
+	WatermarkLow  = "watermark_low"
+	WatermarkHigh = "watermark_high"
+
+	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
+	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
+	EnvCacheExpiry        = "MINIO_CACHE_EXPIRY"
+	EnvCacheMaxUse        = "MINIO_CACHE_MAXUSE"
+	EnvCacheQuota         = "MINIO_CACHE_QUOTA"
+	EnvCacheAfter         = "MINIO_CACHE_AFTER"
+	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
+	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
 
-	EnvCacheDrives              = "MINIO_CACHE_DRIVES"
-	EnvCacheExclude             = "MINIO_CACHE_EXCLUDE"
-	EnvCacheExpiry              = "MINIO_CACHE_EXPIRY"
-	EnvCacheMaxUse              = "MINIO_CACHE_MAXUSE"
-	EnvCacheQuota               = "MINIO_CACHE_QUOTA"
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
-	DefaultExpiry = "90"
-	DefaultQuota  = "80"
+	DefaultExpiry        = "90"
+	DefaultQuota         = "80"
+	DefaultAfter         = "0"
+	DefaultWaterMarkLow  = "70"
+	DefaultWaterMarkHigh = "80"
 )
 
 // DefaultKVS - default KV settings for caching.
@@ -62,6 +72,18 @@ var (
 			Key:   Quota,
 			Value: DefaultQuota,
 		},
+		config.KV{
+			Key:   After,
+			Value: DefaultAfter,
+		},
+		config.KV{
+			Key:   WatermarkLow,
+			Value: DefaultWaterMarkLow,
+		},
+		config.KV{
+			Key:   WatermarkHigh,
+			Value: DefaultWaterMarkHigh,
+		},
 	}
 )
 
@@ -79,7 +101,6 @@ func Enabled(kvs config.KVS) bool {
 // variables and merge them with provided CacheConfiguration.
 func LookupConfig(kvs config.KVS) (Config, error) {
 	cfg := Config{}
-
 	if err := config.CheckValidKeys(config.CacheSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
@@ -121,9 +142,7 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 			return cfg, config.ErrInvalidCacheQuota(err)
 		}
 		cfg.Quota = cfg.MaxUse
-	}
-
-	if quotaStr := env.Get(EnvCacheQuota, kvs.Get(Quota)); quotaStr != "" {
+	} else if quotaStr := env.Get(EnvCacheQuota, kvs.Get(Quota)); quotaStr != "" {
 		cfg.Quota, err = strconv.Atoi(quotaStr)
 		if err != nil {
 			return cfg, config.ErrInvalidCacheQuota(err)
@@ -136,5 +155,45 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		cfg.MaxUse = cfg.Quota
 	}
 
+	if afterStr := env.Get(EnvCacheAfter, kvs.Get(After)); afterStr != "" {
+		cfg.After, err = strconv.Atoi(afterStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheAfter(err)
+		}
+		// after should be a valid value >= 0.
+		if cfg.After < 0 {
+			err := errors.New("cache after value cannot be less than 0")
+			return cfg, config.ErrInvalidCacheAfter(err)
+		}
+	}
+
+	if lowWMStr := env.Get(EnvCacheWatermarkLow, kvs.Get(WatermarkLow)); lowWMStr != "" {
+		cfg.WatermarkLow, err = strconv.Atoi(lowWMStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheWatermarkLow(err)
+		}
+		// WatermarkLow should be a valid percentage.
+		if cfg.WatermarkLow < 0 || cfg.WatermarkLow > 100 {
+			err := errors.New("config min watermark value should be between 0 and 100")
+			return cfg, config.ErrInvalidCacheWatermarkLow(err)
+		}
+	}
+
+	if highWMStr := env.Get(EnvCacheWatermarkHigh, kvs.Get(WatermarkHigh)); highWMStr != "" {
+		cfg.WatermarkHigh, err = strconv.Atoi(highWMStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+		}
+
+		// MaxWatermark should be a valid percentage.
+		if cfg.WatermarkHigh < 0 || cfg.WatermarkHigh > 100 {
+			err := errors.New("config high watermark value should be between 0 and 100")
+			return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+		}
+	}
+	if cfg.WatermarkLow > cfg.WatermarkHigh {
+		err := errors.New("config high watermark value should be greater than low watermark value")
+		return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+	}
 	return cfg, nil
 }

cmd/config/config.go

@@ -24,7 +24,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
@@ -621,6 +621,12 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 	currKVS, ok := c[subSys][tgt]
 	if !ok {
 		currKVS = defaultKVS[subSys]
+	} else {
+		for _, kv := range defaultKVS[subSys] {
+			if _, ok = currKVS.Lookup(kv.Key); !ok {
+				currKVS.Set(kv.Key, kv.Value)
+			}
+		}
 	}
 
 	for _, kv := range kvs {

cmd/config/constants.go

@@ -33,6 +33,10 @@ const (
 	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
 	EnvEndpoints    = "MINIO_ENDPOINTS"
 
+	// API sub-system
+	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
+	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
+
 	EnvUpdate = "MINIO_UPDATE"
 
 	EnvWorm   = "MINIO_WORM"   // legacy

cmd/config/errors-utils.go

@@ -20,7 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
+	"net"
 	"syscall"
 
 	"github.com/minio/minio/pkg/color"
@@ -36,7 +36,17 @@ type Err struct {
 	hint   string
 }
 
-// Return the error message
+// Clone returns a new Err struct with the same information
+func (u Err) Clone() Err {
+	return Err{
+		msg:    u.msg,
+		detail: u.detail,
+		action: u.action,
+		hint:   u.hint,
+	}
+}
+
+// Error returns the error message
 func (u Err) Error() string {
 	if u.detail == "" {
 		if u.msg != "" {
@@ -49,12 +59,16 @@ func (u Err) Error() string {
 
 // Msg - Replace the current error's message
 func (u Err) Msg(m string, args ...interface{}) Err {
-	return Err{
-		msg:    fmt.Sprintf(m, args...),
-		detail: u.detail,
-		action: u.action,
-		hint:   u.hint,
-	}
+	e := u.Clone()
+	e.msg = fmt.Sprintf(m, args...)
+	return e
+}
+
+// Hint - Replace the current error's message
+func (u Err) Hint(m string, args ...interface{}) Err {
+	e := u.Clone()
+	e.hint = fmt.Sprintf(m, args...)
+	return e
 }
 
 // ErrFn function wrapper
@@ -92,9 +106,11 @@ func ErrorToErr(err error) Err {
 	// Show a generic message for known golang errors
 	if errors.Is(err, syscall.EADDRINUSE) {
 		return ErrPortAlreadyInUse(err).Msg("Specified port is already in use")
-	} else if errors.Is(err, syscall.EACCES) {
-		return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
-	} else if os.IsPermission(err) {
+	} else if errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) {
+		switch err.(type) {
+		case *net.OpError:
+			return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
+		}
 		return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
 	} else if errors.Is(err, io.ErrUnexpectedEOF) {
 		return ErrUnexpectedDataContent(err)

cmd/config/errors.go

@@ -66,6 +66,24 @@ var (
 		"MINIO_CACHE_QUOTA: Valid cache quota value must be between 0-100",
 	)
 
+	ErrInvalidCacheAfter = newErrFn(
+		"Invalid cache after value",
+		"Please check the passed value",
+		"MINIO_CACHE_AFTER: Valid cache after value must be 0 or greater",
+	)
+
+	ErrInvalidCacheWatermarkLow = newErrFn(
+		"Invalid cache low watermark value",
+		"Please check the passed value",
+		"MINIO_CACHE_WATERMARK_LOW: Valid cache low watermark value must be between 0-100",
+	)
+
+	ErrInvalidCacheWatermarkHigh = newErrFn(
+		"Invalid cache high watermark value",
+		"Please check the passed value",
+		"MINIO_CACHE_WATERMARK_HIGH: Valid cache high watermark value must be between 0-100",
+	)
+
 	ErrInvalidCacheEncryptionKey = newErrFn(
 		"Invalid cache encryption master key value",
 		"Please check the passed value",

cmd/config/etcd/dns/etcd_dns.go

@@ -38,18 +38,18 @@ var ErrNoEntriesFound = errors.New("No entries found for this key")
 const etcdPathSeparator = "/"
 
 // create a new coredns service record for the bucket.
-func newCoreDNSMsg(ip string, port string, ttl uint32) ([]byte, error) {
+func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, error) {
 	return json.Marshal(&SrvRecord{
 		Host:         ip,
 		Port:         json.Number(port),
 		TTL:          ttl,
-		CreationDate: time.Now().UTC(),
+		CreationDate: t,
 	})
 }
 
 // List - Retrieves list of DNS entries for the domain.
-func (c *CoreDNS) List() ([]SrvRecord, error) {
-	var srvRecords []SrvRecord
+func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
+	var srvRecords = map[string][]SrvRecord{}
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.", domainName), c.prefixPath)
 		records, err := c.list(key)
@@ -60,7 +60,7 @@ func (c *CoreDNS) List() ([]SrvRecord, error) {
 			if record.Key == "" {
 				continue
 			}
-			srvRecords = append(srvRecords, record)
+			srvRecords[record.Key] = append(srvRecords[record.Key], record)
 		}
 	}
 	return srvRecords, nil
@@ -117,9 +117,6 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 		if err != nil {
 			return nil, err
 		}
-		if r.Count == 0 {
-			return nil, ErrNoEntriesFound
-		}
 	}
 
 	var srvRecords []SrvRecord
@@ -143,9 +140,6 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 		srvRecords = append(srvRecords, srvRecord)
 
 	}
-	if len(srvRecords) == 0 {
-		return nil, ErrNoEntriesFound
-	}
 	sort.Slice(srvRecords, func(i int, j int) bool {
 		return srvRecords[i].Key < srvRecords[j].Key
 	})
@@ -156,8 +150,9 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 func (c *CoreDNS) Put(bucket string) error {
 	c.Delete(bucket) // delete any existing entries.
 
+	t := time.Now().UTC()
 	for ip := range c.domainIPs {
-		bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL)
+		bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL, t)
 		if err != nil {
 			return err
 		}

cmd/config/identity/ldap/config.go

@@ -21,7 +21,8 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	"regexp"
+	"net"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/config"
@@ -44,34 +45,45 @@ type Config struct {
 	STSExpiryDuration string `json:"stsExpiryDuration"`
 
 	// Format string for usernames
-	UsernameFormat string `json:"usernameFormat"`
+	UsernameFormat        string   `json:"usernameFormat"`
+	UsernameFormats       []string `json:"-"`
+	UsernameSearchFilter  string   `json:"-"`
+	UsernameSearchBaseDNS []string `json:"-"`
 
-	GroupSearchBaseDN  string `json:"groupSearchBaseDN"`
-	GroupSearchFilter  string `json:"groupSearchFilter"`
-	GroupNameAttribute string `json:"groupNameAttribute"`
+	GroupSearchBaseDN  string   `json:"groupSearchBaseDN"`
+	GroupSearchBaseDNS []string `json:"-"`
+	GroupSearchFilter  string   `json:"groupSearchFilter"`
+	GroupNameAttribute string   `json:"groupNameAttribute"`
 
 	stsExpiryDuration time.Duration // contains converted value
 	tlsSkipVerify     bool          // allows skipping TLS verification
+	serverInsecure    bool          // allows plain text connection to LDAP Server
 	rootCAs           *x509.CertPool
 }
 
 // LDAP keys and envs.
 const (
-	ServerAddr         = "server_addr"
-	STSExpiry          = "sts_expiry"
-	UsernameFormat     = "username_format"
-	GroupSearchFilter  = "group_search_filter"
-	GroupNameAttribute = "group_name_attribute"
-	GroupSearchBaseDN  = "group_search_base_dn"
-	TLSSkipVerify      = "tls_skip_verify"
+	ServerAddr           = "server_addr"
+	STSExpiry            = "sts_expiry"
+	UsernameFormat       = "username_format"
+	UsernameSearchFilter = "username_search_filter"
+	UsernameSearchBaseDN = "username_search_base_dn"
+	GroupSearchFilter    = "group_search_filter"
+	GroupNameAttribute   = "group_name_attribute"
+	GroupSearchBaseDN    = "group_search_base_dn"
+	TLSSkipVerify        = "tls_skip_verify"
+	ServerInsecure       = "server_insecure"
 
-	EnvServerAddr         = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
-	EnvSTSExpiry          = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
-	EnvTLSSkipVerify      = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"
-	EnvUsernameFormat     = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
-	EnvGroupSearchFilter  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
-	EnvGroupNameAttribute = "MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE"
-	EnvGroupSearchBaseDN  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
+	EnvServerAddr           = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
+	EnvSTSExpiry            = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
+	EnvTLSSkipVerify        = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"
+	EnvServerInsecure       = "MINIO_IDENTITY_LDAP_SERVER_INSECURE"
+	EnvUsernameFormat       = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
+	EnvUsernameSearchFilter = "MINIO_IDENTITY_LDAP_USERNAME_SEARCH_FILTER"
+	EnvUsernameSearchBaseDN = "MINIO_IDENTITY_LDAP_USERNAME_SEARCH_BASE_DN"
+	EnvGroupSearchFilter    = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
+	EnvGroupNameAttribute   = "MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE"
+	EnvGroupSearchBaseDN    = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
 )
 
 // DefaultKVS - default config for LDAP config
@@ -82,11 +94,15 @@ var (
 			Value: "",
 		},
 		config.KV{
-			Key:   STSExpiry,
-			Value: "1h",
+			Key:   UsernameFormat,
+			Value: "",
 		},
 		config.KV{
-			Key:   UsernameFormat,
+			Key:   UsernameSearchFilter,
+			Value: "",
+		},
+		config.KV{
+			Key:   UsernameSearchBaseDN,
 			Value: "",
 		},
 		config.KV{
@@ -101,19 +117,146 @@ var (
 			Key:   GroupSearchBaseDN,
 			Value: "",
 		},
+		config.KV{
+			Key:   STSExpiry,
+			Value: "1h",
+		},
 		config.KV{
 			Key:   TLSSkipVerify,
 			Value: config.EnableOff,
 		},
+		config.KV{
+			Key:   ServerInsecure,
+			Value: config.EnableOff,
+		},
 	}
 )
 
+const (
+	dnDelimiter = ";"
+)
+
+func getGroups(conn *ldap.Conn, sreq *ldap.SearchRequest) ([]string, error) {
+	var groups []string
+	sres, err := conn.Search(sreq)
+	if err != nil {
+		return nil, err
+	}
+	for _, entry := range sres.Entries {
+		// We only queried one attribute,
+		// so we only look up the first one.
+		groups = append(groups, entry.Attributes[0].Values...)
+	}
+	return groups, nil
+}
+
+func (l *Config) bind(conn *ldap.Conn, username, password string) ([]string, error) {
+	var bindDNS = make([]string, len(l.UsernameFormats))
+	for i, usernameFormat := range l.UsernameFormats {
+		bindDN := fmt.Sprintf(usernameFormat, username)
+		// Bind with user credentials to validate the password
+		if err := conn.Bind(bindDN, password); err != nil {
+			return nil, err
+		}
+		bindDNS[i] = bindDN
+	}
+	return bindDNS, nil
+}
+
+var standardAttributes = []string{
+	"givenName",
+	"sn",
+	"cn",
+	"memberOf",
+	"email",
+}
+
+// Bind - binds to ldap, searches LDAP and returns list of groups.
+func (l *Config) Bind(username, password string) ([]string, error) {
+	conn, err := l.Connect()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	bindDNS, err := l.bind(conn, username, password)
+	if err != nil {
+		return nil, err
+	}
+
+	var groups []string
+	if l.UsernameSearchFilter != "" {
+		for _, userSearchBase := range l.UsernameSearchBaseDNS {
+			filter := strings.Replace(l.UsernameSearchFilter, "%s",
+				ldap.EscapeFilter(username), -1)
+
+			searchRequest := ldap.NewSearchRequest(
+				userSearchBase,
+				ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+				filter,
+				standardAttributes,
+				nil,
+			)
+
+			groups, err = getGroups(conn, searchRequest)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	if l.GroupSearchFilter != "" {
+		for _, groupSearchBase := range l.GroupSearchBaseDNS {
+			var filters []string
+			if l.GroupNameAttribute == "" {
+				filters = []string{strings.Replace(l.GroupSearchFilter, "%s",
+					ldap.EscapeFilter(username), -1)}
+			} else {
+				// With group name attribute specified, make sure to
+				// include search queries for CN distinguished name
+				for _, bindDN := range bindDNS {
+					filters = append(filters, strings.Replace(l.GroupSearchFilter, "%s",
+						ldap.EscapeFilter(bindDN), -1))
+				}
+			}
+			for _, filter := range filters {
+				searchRequest := ldap.NewSearchRequest(
+					groupSearchBase,
+					ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+					filter,
+					standardAttributes,
+					nil,
+				)
+
+				var newGroups []string
+				newGroups, err = getGroups(conn, searchRequest)
+				if err != nil {
+					return nil, err
+				}
+
+				groups = append(groups, newGroups...)
+			}
+		}
+	}
+
+	return groups, nil
+}
+
 // Connect connect to ldap server.
 func (l *Config) Connect() (ldapConn *ldap.Conn, err error) {
 	if l == nil {
-		// Happens when LDAP is not configured.
-		return
+		return nil, errors.New("LDAP is not configured")
 	}
+
+	if _, _, err = net.SplitHostPort(l.ServerAddr); err != nil {
+		// User default LDAP port if none specified "636"
+		l.ServerAddr = net.JoinHostPort(l.ServerAddr, "636")
+	}
+
+	if l.serverInsecure {
+		return ldap.Dial("tcp", l.ServerAddr)
+	}
+
 	return ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{
 		InsecureSkipVerify: l.tlsSkipVerify,
 		RootCAs:            l.rootCAs,
@@ -154,6 +297,12 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 		l.STSExpiryDuration = v
 		l.stsExpiryDuration = expDur
 	}
+	if v := env.Get(EnvServerInsecure, kvs.Get(ServerInsecure)); v != "" {
+		l.serverInsecure, err = config.ParseBool(v)
+		if err != nil {
+			return l, err
+		}
+	}
 	if v := env.Get(EnvTLSSkipVerify, kvs.Get(TLSSkipVerify)); v != "" {
 		l.tlsSkipVerify, err = config.ParseBool(v)
 		if err != nil {
@@ -161,102 +310,44 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 		}
 	}
 	if v := env.Get(EnvUsernameFormat, kvs.Get(UsernameFormat)); v != "" {
-		subs, err := NewSubstituter("username", "test")
-		if err != nil {
-			return l, err
+		if !strings.Contains(v, "%s") {
+			return l, errors.New("LDAP username format doesn't have '%s' substitution")
 		}
-		if _, err := subs.Substitute(v); err != nil {
-			return l, err
-		}
-		l.UsernameFormat = v
+		l.UsernameFormats = strings.Split(v, dnDelimiter)
 	} else {
 		return l, fmt.Errorf("'%s' cannot be empty and must have a value", UsernameFormat)
 	}
 
+	if v := env.Get(EnvUsernameSearchFilter, kvs.Get(UsernameSearchFilter)); v != "" {
+		if !strings.Contains(v, "%s") {
+			return l, errors.New("LDAP username search filter doesn't have '%s' substitution")
+		}
+		l.UsernameSearchFilter = v
+	}
+
+	if v := env.Get(EnvUsernameSearchBaseDN, kvs.Get(UsernameSearchBaseDN)); v != "" {
+		l.UsernameSearchBaseDNS = strings.Split(v, dnDelimiter)
+	}
+
 	grpSearchFilter := env.Get(EnvGroupSearchFilter, kvs.Get(GroupSearchFilter))
 	grpSearchNameAttr := env.Get(EnvGroupNameAttribute, kvs.Get(GroupNameAttribute))
 	grpSearchBaseDN := env.Get(EnvGroupSearchBaseDN, kvs.Get(GroupSearchBaseDN))
 
 	// Either all group params must be set or none must be set.
-	allNotSet := grpSearchFilter == "" && grpSearchNameAttr == "" && grpSearchBaseDN == ""
-	allSet := grpSearchFilter != "" && grpSearchNameAttr != "" && grpSearchBaseDN != ""
-	if !allNotSet && !allSet {
-		return l, errors.New("All group related parameters must be set")
+	var allSet bool
+	if grpSearchFilter != "" {
+		if grpSearchNameAttr == "" || grpSearchBaseDN == "" {
+			return l, errors.New("All group related parameters must be set")
+		}
+		allSet = true
 	}
 
 	if allSet {
-		subs, err := NewSubstituter("username", "test", "usernamedn", "test2")
-		if err != nil {
-			return l, err
-		}
-		if _, err := subs.Substitute(grpSearchFilter); err != nil {
-			return l, fmt.Errorf("Only username and usernamedn may be substituted in the group search filter string: %s", err)
-		}
 		l.GroupSearchFilter = grpSearchFilter
 		l.GroupNameAttribute = grpSearchNameAttr
-		subs, err = NewSubstituter("username", "test", "usernamedn", "test2")
-		if err != nil {
-			return l, err
-		}
-		if _, err := subs.Substitute(grpSearchBaseDN); err != nil {
-			return l, fmt.Errorf("Only username and usernamedn may be substituted in the base DN string: %s", err)
-		}
-		l.GroupSearchBaseDN = grpSearchBaseDN
+		l.GroupSearchBaseDNS = strings.Split(grpSearchBaseDN, dnDelimiter)
 	}
 
 	l.rootCAs = rootCAs
 	return l, nil
 }
-
-// Substituter - This type is to allow restricted runtime
-// substitutions of variables in LDAP configuration items during
-// runtime.
-type Substituter struct {
-	vals map[string]string
-}
-
-// NewSubstituter - sets up the substituter for usage, for e.g.:
-//
-//  subber := NewSubstituter("username", "john")
-func NewSubstituter(v ...string) (Substituter, error) {
-	if len(v)%2 != 0 {
-		return Substituter{}, errors.New("Need an even number of arguments")
-	}
-	vals := make(map[string]string)
-	for i := 0; i < len(v); i += 2 {
-		vals[v[i]] = v[i+1]
-	}
-	return Substituter{vals: vals}, nil
-}
-
-// Substitute - performs substitution on the given string `t`. Returns
-// an error if there are any variables in the input that do not have
-// values in the substituter. E.g.:
-//
-// subber.Substitute("uid=${username},cn=users,dc=example,dc=com")
-//
-// or
-//
-// subber.Substitute("uid={username},cn=users,dc=example,dc=com")
-//
-// returns "uid=john,cn=users,dc=example,dc=com"
-//
-// whereas:
-//
-// subber.Substitute("uid=${usernamedn}")
-//
-// returns an error.
-func (s *Substituter) Substitute(t string) (string, error) {
-	for k, v := range s.vals {
-		reDollar := regexp.MustCompile(fmt.Sprintf(`\$\{%s\}`, k))
-		t = reDollar.ReplaceAllLiteralString(t, v)
-		reFlower := regexp.MustCompile(fmt.Sprintf(`\{%s\}`, k))
-		t = reFlower.ReplaceAllLiteralString(t, v)
-	}
-	// Check if all requested substitutions have been made.
-	re := regexp.MustCompile(`\{.*\}`)
-	if re.MatchString(t) {
-		return "", errors.New("unsupported substitution requested")
-	}
-	return t, nil
-}

cmd/config/identity/ldap/config_test.go

@@ -1,93 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ldap
-
-import (
-	"testing"
-)
-
-func TestSubstituter(t *testing.T) {
-	tests := []struct {
-		KV               []string
-		SubstitutableStr string
-		SubstitutedStr   string
-		ErrExpected      bool
-	}{
-		{
-			KV:               []string{"username", "john"},
-			SubstitutableStr: "uid=${username},cn=users,dc=example,dc=com",
-			SubstitutedStr:   "uid=john,cn=users,dc=example,dc=com",
-			ErrExpected:      false,
-		},
-		{
-			KV:               []string{"username", "john"},
-			SubstitutableStr: "uid={username},cn=users,dc=example,dc=com",
-			SubstitutedStr:   "uid=john,cn=users,dc=example,dc=com",
-			ErrExpected:      false,
-		},
-		{
-			KV:               []string{"username", "john"},
-			SubstitutableStr: "(&(objectclass=group)(member=${username}))",
-			SubstitutedStr:   "(&(objectclass=group)(member=john))",
-			ErrExpected:      false,
-		},
-		{
-			KV:               []string{"username", "john"},
-			SubstitutableStr: "(&(objectclass=group)(member={username}))",
-			SubstitutedStr:   "(&(objectclass=group)(member=john))",
-			ErrExpected:      false,
-		},
-		{
-			KV:               []string{"username", "john"},
-			SubstitutableStr: "uid=${{username}},cn=users,dc=example,dc=com",
-			ErrExpected:      true,
-		},
-		{
-			KV:               []string{"username", "john"},
-			SubstitutableStr: "uid=${usernamedn},cn=users,dc=example,dc=com",
-			ErrExpected:      true,
-		},
-		{
-			KV:               []string{"username"},
-			SubstitutableStr: "uid=${usernamedn},cn=users,dc=example,dc=com",
-			ErrExpected:      true,
-		},
-		{
-			KV:               []string{"username", "john"},
-			SubstitutableStr: "(&(objectclass=user)(sAMAccountName={username})(memberOf=CN=myorg,OU=Rialto,OU=Application Managed,OU=Groups,DC=amr,DC=corp,DC=myorg,DC=com))",
-			SubstitutedStr:   "(&(objectclass=user)(sAMAccountName=john)(memberOf=CN=myorg,OU=Rialto,OU=Application Managed,OU=Groups,DC=amr,DC=corp,DC=myorg,DC=com))",
-			ErrExpected:      false,
-		},
-	}
-
-	for _, test := range tests {
-		test := test
-		t.Run(test.SubstitutableStr, func(t *testing.T) {
-			subber, err := NewSubstituter(test.KV...)
-			if err != nil && !test.ErrExpected {
-				t.Errorf("Unexpected failure %s", err)
-			}
-			gotStr, err := subber.Substitute(test.SubstitutableStr)
-			if err != nil && !test.ErrExpected {
-				t.Errorf("Unexpected failure %s", err)
-			}
-			if gotStr != test.SubstitutedStr {
-				t.Errorf("Expected %s, got %s", test.SubstitutedStr, gotStr)
-			}
-		})
-	}
-}

cmd/config/identity/ldap/help.go

@@ -28,24 +28,33 @@ var (
 		},
 		config.HelpKV{
 			Key:         UsernameFormat,
-			Description: `username bind DNs e.g. "uid=%s,cn=accounts,dc=myldapserver,dc=com"`,
+			Description: `";" separated list of username bind DNs e.g. "uid=%s,cn=accounts,dc=myldapserver,dc=com"`,
+			Type:        "list",
+		},
+		config.HelpKV{
+			Key:         UsernameSearchFilter,
+			Description: `user search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)"`,
 			Type:        "string",
 		},
 		config.HelpKV{
 			Key:         GroupSearchFilter,
 			Description: `search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"`,
-			Optional:    true,
-			Type:        "string",
-		},
-		config.HelpKV{
-			Key:         GroupNameAttribute,
-			Description: `search attribute for group name e.g. "cn"`,
-			Optional:    true,
 			Type:        "string",
 		},
 		config.HelpKV{
 			Key:         GroupSearchBaseDN,
-			Description: `group search base DNs e.g. "dc=myldapserver,dc=com"`,
+			Description: `";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"`,
+			Type:        "list",
+		},
+		config.HelpKV{
+			Key:         UsernameSearchBaseDN,
+			Description: `";" separated list of username search DNs`,
+			Type:        "list",
+			Optional:    true,
+		},
+		config.HelpKV{
+			Key:         GroupNameAttribute,
+			Description: `search attribute for group name e.g. "cn"`,
 			Optional:    true,
 			Type:        "string",
 		},
@@ -57,7 +66,13 @@ var (
 		},
 		config.HelpKV{
 			Key:         TLSSkipVerify,
-			Description: `trust server TLS without verification, defaults to "on" (verify)`,
+			Description: `trust server TLS without verification, defaults to "off" (verify)`,
+			Optional:    true,
+			Type:        "on|off",
+		},
+		config.HelpKV{
+			Key:         ServerInsecure,
+			Description: `allow plain text connection to AD/LDAP server, defaults to "off"`,
 			Optional:    true,
 			Type:        "on|off",
 		},

cmd/config/identity/openid/help.go

@@ -32,9 +32,15 @@ var (
 			Type:        "string",
 			Optional:    true,
 		},
+		config.HelpKV{
+			Key:         ClaimName,
+			Description: `JWT canned policy claim name, defaults to "policy"`,
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         ClaimPrefix,
-			Description: `JWT claim namespace prefix e.g. "customer1"`,
+			Description: `JWT claim namespace prefix e.g. "customer1/"`,
 			Optional:    true,
 			Type:        "string",
 		},

cmd/config/identity/openid/jwt.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	xnet "github.com/minio/minio/pkg/net"
 )
 
@@ -41,6 +42,7 @@ type Config struct {
 	} `json:"jwks"`
 	URL          *xnet.URL `json:"url,omitempty"`
 	ClaimPrefix  string    `json:"claimPrefix,omitempty"`
+	ClaimName    string    `json:"claimName,omitempty"`
 	DiscoveryDoc DiscoveryDoc
 	ClientID     string
 	publicKeys   map[string]crypto.PublicKey
@@ -209,12 +211,14 @@ func (p *JWT) ID() ID {
 const (
 	JwksURL     = "jwks_url"
 	ConfigURL   = "config_url"
+	ClaimName   = "claim_name"
 	ClaimPrefix = "claim_prefix"
 	ClientID    = "client_id"
 
 	EnvIdentityOpenIDClientID    = "MINIO_IDENTITY_OPENID_CLIENT_ID"
 	EnvIdentityOpenIDJWKSURL     = "MINIO_IDENTITY_OPENID_JWKS_URL"
 	EnvIdentityOpenIDURL         = "MINIO_IDENTITY_OPENID_CONFIG_URL"
+	EnvIdentityOpenIDClaimName   = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
 	EnvIdentityOpenIDClaimPrefix = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
 )
 
@@ -247,6 +251,7 @@ func parseDiscoveryDoc(u *xnet.URL, transport *http.Transport, closeRespFn func(
 	}
 	resp, err := clnt.Do(req)
 	if err != nil {
+		clnt.CloseIdleConnections()
 		return d, err
 	}
 	defer closeRespFn(resp.Body)
@@ -271,6 +276,10 @@ var (
 			Key:   ClientID,
 			Value: "",
 		},
+		config.KV{
+			Key:   ClaimName,
+			Value: iampolicy.PolicyName,
+		},
 		config.KV{
 			Key:   ClaimPrefix,
 			Value: "",
@@ -299,6 +308,7 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 	}
 
 	c = Config{
+		ClaimName:   env.Get(EnvIdentityOpenIDClaimName, kvs.Get(ClaimName)),
 		ClaimPrefix: env.Get(EnvIdentityOpenIDClaimPrefix, kvs.Get(ClaimPrefix)),
 		publicKeys:  make(map[string]crypto.PublicKey),
 		ClientID:    env.Get(EnvIdentityOpenIDClientID, kvs.Get(ClientID)),
@@ -317,6 +327,11 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 			return c, err
 		}
 	}
+
+	if c.ClaimName == "" {
+		c.ClaimName = iampolicy.PolicyName
+	}
+
 	if jwksURL == "" {
 		// Fallback to discovery document jwksURL
 		jwksURL = c.DiscoveryDoc.JwksURI
@@ -330,9 +345,11 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 	if err != nil {
 		return c, err
 	}
+
 	if err = c.PopulatePublicKey(); err != nil {
 		return c, err
 	}
+
 	return c, nil
 }
 

cmd/config/notify/help.go

@@ -165,6 +165,12 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         target.KafkaSASLMechanism,
+			Description: "sasl authentication mechanism, default 'plain'",
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         target.KafkaTLSClientAuth,
 			Description: "clientAuth determines the Kafka server's policy for TLS client auth",
@@ -213,6 +219,12 @@ var (
 			Optional:    true,
 			Type:        "number",
 		},
+		config.HelpKV{
+			Key:         target.KafkaVersion,
+			Description: "specify the version of the Kafka cluster",
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,
@@ -352,6 +364,7 @@ var (
 		config.HelpKV{
 			Key:         target.MySQLDSNString,
 			Description: "MySQL data-source-name connection string",
+			Optional:    true,
 			Type:        "string",
 		},
 		config.HelpKV{

cmd/config/notify/parse.go

@@ -42,7 +42,14 @@ const (
 func TestNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport,
 	targetIDs []event.TargetID) error {
 	test := true
-	_, err := RegisterNotificationTargets(cfg, doneCh, transport, targetIDs, test)
+	targets, err := RegisterNotificationTargets(cfg, doneCh, transport, targetIDs, test)
+	if err == nil {
+		// Close all targets since we are only testing connections.
+		for _, t := range targets.TargetMap() {
+			_ = t.Close()
+		}
+	}
+
 	return err
 }
 
@@ -58,9 +65,19 @@ func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport
 // * Add a new target in pkg/event/target package.
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
-func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool) (*event.TargetList, error) {
+func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool) (_ *event.TargetList, err error) {
 	targetList := event.NewTargetList()
-	if err := checkValidNotificationKeys(cfg); err != nil {
+
+	defer func() {
+		// Automatically close all connections to targets when an error occur
+		if err != nil {
+			for _, t := range targetList.TargetMap() {
+				_ = t.Close()
+			}
+		}
+	}()
+
+	if err = checkValidNotificationKeys(cfg); err != nil {
 		return nil, err
 	}
 
@@ -118,23 +135,20 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewAMQPTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewAMQPTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range esTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewElasticsearchTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewElasticsearchTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 
@@ -142,9 +156,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range kafkaTargets {
@@ -152,16 +163,13 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 			continue
 		}
 		args.TLS.RootCAs = transport.TLSClientConfig.RootCAs
-		newTarget, err := target.NewKafkaTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewKafkaTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range mqttTargets {
@@ -169,113 +177,91 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 			continue
 		}
 		args.RootCAs = transport.TLSClientConfig.RootCAs
-		newTarget, err := target.NewMQTTTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewMQTTTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range mysqlTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewMySQLTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewMySQLTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range natsTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewNATSTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewNATSTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range nsqTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewNSQTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewNSQTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range postgresTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewPostgreSQLTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewPostgreSQLTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range redisTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewRedisTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewRedisTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range webhookTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewWebhookTarget(id, args, doneCh, logger.LogOnceIf, transport)
+		newTarget, err := target.NewWebhookTarget(id, args, doneCh, logger.LogOnceIf, transport, test)
 		if err != nil {
 			return nil, err
 		}
 		if err := targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-			continue
-		}
 	}
 
 	if test {
@@ -366,6 +352,10 @@ var (
 			Key:   target.KafkaSASLPassword,
 			Value: "",
 		},
+		config.KV{
+			Key:   target.KafkaSASLMechanism,
+			Value: "plain",
+		},
 		config.KV{
 			Key:   target.KafkaClientTLSCert,
 			Value: "",
@@ -398,6 +388,10 @@ var (
 			Key:   target.KafkaQueueDir,
 			Value: "",
 		},
+		config.KV{
+			Key:   target.KafkaVersion,
+			Value: "",
+		},
 	}
 )
 
@@ -465,12 +459,18 @@ func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs
 			queueDirEnv = queueDirEnv + config.Default + k
 		}
 
+		versionEnv := target.EnvKafkaVersion
+		if k != config.Default {
+			versionEnv = versionEnv + config.Default + k
+		}
+
 		kafkaArgs := target.KafkaArgs{
 			Enable:     enabled,
 			Brokers:    brokers,
 			Topic:      env.Get(topicEnv, kv.Get(target.KafkaTopic)),
 			QueueDir:   env.Get(queueDirEnv, kv.Get(target.KafkaQueueDir)),
 			QueueLimit: queueLimit,
+			Version:    env.Get(versionEnv, kv.Get(target.KafkaVersion)),
 		}
 
 		tlsEnableEnv := target.EnvKafkaTLS
@@ -511,9 +511,14 @@ func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs
 		if k != config.Default {
 			saslPasswordEnv = saslPasswordEnv + config.Default + k
 		}
+		saslMechanismEnv := target.EnvKafkaSASLMechanism
+		if k != config.Default {
+			saslMechanismEnv = saslMechanismEnv + config.Default + k
+		}
 		kafkaArgs.SASL.Enable = env.Get(saslEnableEnv, kv.Get(target.KafkaSASL)) == config.EnableOn
 		kafkaArgs.SASL.User = env.Get(saslUsernameEnv, kv.Get(target.KafkaSASLUsername))
 		kafkaArgs.SASL.Password = env.Get(saslPasswordEnv, kv.Get(target.KafkaSASLPassword))
+		kafkaArgs.SASL.Mechanism = env.Get(saslMechanismEnv, kv.Get(target.KafkaSASLMechanism))
 
 		if err = kafkaArgs.Validate(); err != nil {
 			return nil, err

cmd/config/policy/opa/config.go

@@ -223,5 +223,6 @@ func (o *Opa) IsAllowed(args iampolicy.Args) (bool, error) {
 		}
 		return resultAllow.Result.Allow, nil
 	}
+
 	return result.Result, nil
 }

cmd/consolelogger.go

@@ -24,7 +24,6 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/cmd/logger/target/console"
-	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/pubsub"
 )
@@ -84,17 +83,20 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan s
 	cnt := 0
 	// by default send all console logs in the ring buffer unless node or limit query parameters
 	// are set.
-	var lastN []madmin.LogInfo
+	var lastN []log.Info
 	if last > defaultLogBufferCount || last <= 0 {
 		last = defaultLogBufferCount
 	}
 
-	lastN = make([]madmin.LogInfo, last)
+	lastN = make([]log.Info, last)
 	sys.RLock()
 	sys.logBuf.Do(func(p interface{}) {
-		if p != nil && (p.(madmin.LogInfo)).SendLog(node, logKind) {
-			lastN[cnt%last] = p.(madmin.LogInfo)
-			cnt++
+		if p != nil {
+			lg, ok := p.(log.Info)
+			if ok && lg.SendLog(node, logKind) {
+				lastN[cnt%last] = lg
+				cnt++
+			}
 		}
 	})
 	sys.RUnlock()
@@ -102,7 +104,7 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan s
 	if cnt > 0 {
 		for i := 0; i < last; i++ {
 			entry := lastN[(cnt+i)%last]
-			if (entry == madmin.LogInfo{}) {
+			if (entry == log.Info{}) {
 				continue
 			}
 			select {
@@ -118,12 +120,12 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan s
 // Send log message 'e' to console and publish to console
 // log pubsub system
 func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {
-	var lg madmin.LogInfo
+	var lg log.Info
 	switch e := e.(type) {
 	case log.Entry:
-		lg = madmin.LogInfo{Entry: e, NodeName: sys.nodeName}
+		lg = log.Info{Entry: e, NodeName: sys.nodeName}
 	case string:
-		lg = madmin.LogInfo{ConsoleMsg: e, NodeName: sys.nodeName}
+		lg = log.Info{ConsoleMsg: e, NodeName: sys.nodeName}
 	}
 
 	sys.pubsub.Publish(lg)

cmd/crypto/config.go

@@ -16,7 +16,6 @@ package crypto
 
 import (
 	"errors"
-	"fmt"
 	"net/http"
 	"reflect"
 	"strconv"
@@ -342,7 +341,7 @@ func LookupVaultConfig(kvs config.KVS) (VaultConfig, error) {
 	if keyVersion := env.Get(EnvKMSVaultKeyVersion, kvs.Get(KMSVaultKeyVersion)); keyVersion != "" {
 		vcfg.Key.Version, err = strconv.Atoi(keyVersion)
 		if err != nil {
-			return vcfg, fmt.Errorf("Unable to parse VaultKeyVersion value (`%s`)", keyVersion)
+			return vcfg, Errorf("Unable to parse VaultKeyVersion value (`%s`)", keyVersion)
 		}
 	}
 

cmd/crypto/error.go

@@ -14,60 +14,79 @@
 
 package crypto
 
-import "errors"
+import (
+	"fmt"
+)
 
 // Error is the generic type for any error happening during decrypting
 // an object. It indicates that the object itself or its metadata was
 // modified accidentally or maliciously.
-type Error string
+type Error struct {
+	err error
+}
 
-func (e Error) Error() string { return string(e) }
+// Errorf - formats according to a format specifier and returns
+// the string as a value that satisfies error of type crypto.Error
+func Errorf(format string, a ...interface{}) error {
+	return Error{err: fmt.Errorf(format, a...)}
+}
+
+// Unwrap the internal error.
+func (e Error) Unwrap() error { return e.err }
+
+// Error 'error' compatible method.
+func (e Error) Error() string {
+	if e.err == nil {
+		return "crypto: cause <nil>"
+	}
+	return e.err.Error()
+}
 
 var (
 	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
 	// is not supported.
-	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+	ErrInvalidEncryptionMethod = Errorf("The encryption method is not supported")
 
 	// ErrInvalidCustomerAlgorithm indicates that the specified SSE-C algorithm
 	// is not supported.
-	ErrInvalidCustomerAlgorithm = errors.New("The SSE-C algorithm is not supported")
+	ErrInvalidCustomerAlgorithm = Errorf("The SSE-C algorithm is not supported")
 
 	// ErrMissingCustomerKey indicates that the HTTP headers contains no SSE-C client key.
-	ErrMissingCustomerKey = errors.New("The SSE-C request is missing the customer key")
+	ErrMissingCustomerKey = Errorf("The SSE-C request is missing the customer key")
 
 	// ErrMissingCustomerKeyMD5 indicates that the HTTP headers contains no SSE-C client key
 	// MD5 checksum.
-	ErrMissingCustomerKeyMD5 = errors.New("The SSE-C request is missing the customer key MD5")
+	ErrMissingCustomerKeyMD5 = Errorf("The SSE-C request is missing the customer key MD5")
 
 	// ErrInvalidCustomerKey indicates that the SSE-C client key is not valid - e.g. not a
 	// base64-encoded string or not 256 bits long.
-	ErrInvalidCustomerKey = errors.New("The SSE-C client key is invalid")
+	ErrInvalidCustomerKey = Errorf("The SSE-C client key is invalid")
 
 	// ErrSecretKeyMismatch indicates that the provided secret key (SSE-C client key / SSE-S3 KMS key)
 	// does not match the secret key used during encrypting the object.
-	ErrSecretKeyMismatch = errors.New("The secret key does not match the secret key used during upload")
+	ErrSecretKeyMismatch = Errorf("The secret key does not match the secret key used during upload")
 
 	// ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the
 	// computed MD5 sum. This means that the client provided either the wrong key for
 	// a certain MD5 checksum or the wrong MD5 for a certain key.
-	ErrCustomerKeyMD5Mismatch = errors.New("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
+	ErrCustomerKeyMD5Mismatch = Errorf("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
 	// ErrIncompatibleEncryptionMethod indicates that both SSE-C headers and SSE-S3 headers were specified, and are incompatible
 	// The client needs to remove the SSE-S3 header or the SSE-C headers
-	ErrIncompatibleEncryptionMethod = errors.New("Server side encryption specified with both SSE-C and SSE-S3 headers")
+	ErrIncompatibleEncryptionMethod = Errorf("Server side encryption specified with both SSE-C and SSE-S3 headers")
 )
 
-const (
-	errMissingInternalIV            Error = "The object metadata is missing the internal encryption IV"
-	errMissingInternalSealAlgorithm Error = "The object metadata is missing the internal seal algorithm"
+var (
+	errMissingInternalIV            = Errorf("The object metadata is missing the internal encryption IV")
+	errMissingInternalSealAlgorithm = Errorf("The object metadata is missing the internal seal algorithm")
 
-	errInvalidInternalIV            Error = "The internal encryption IV is malformed"
-	errInvalidInternalSealAlgorithm Error = "The internal seal algorithm is invalid and not supported"
+	errInvalidInternalIV            = Errorf("The internal encryption IV is malformed")
+	errInvalidInternalSealAlgorithm = Errorf("The internal seal algorithm is invalid and not supported")
 
-	errMissingUpdatedKey Error = "The key update returned no error but also no sealed key"
+	errMissingUpdatedKey = Errorf("The key update returned no error but also no sealed key")
 )
 
 var (
 	// errOutOfEntropy indicates that the a source of randomness (PRNG) wasn't able
 	// to produce enough random data. This is fatal error and should cause a panic.
-	errOutOfEntropy = errors.New("Unable to read enough randomness from the system")
+	errOutOfEntropy = Errorf("Unable to read enough randomness from the system")
 )

cmd/crypto/kes.go

@@ -19,7 +19,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -76,13 +75,13 @@ type KesConfig struct {
 func (k KesConfig) Verify() (err error) {
 	switch {
 	case k.Endpoint == "":
-		err = errors.New("crypto: missing kes endpoint")
+		err = Errorf("crypto: missing kes endpoint")
 	case k.CertFile == "":
-		err = errors.New("crypto: missing cert file")
+		err = Errorf("crypto: missing cert file")
 	case k.KeyFile == "":
-		err = errors.New("crypto: missing key file")
+		err = Errorf("crypto: missing key file")
 	case k.DefaultKeyID == "":
-		err = errors.New("crypto: missing default key id")
+		err = Errorf("crypto: missing default key id")
 	}
 	return err
 }
@@ -153,7 +152,7 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 		return key, nil, err
 	}
 	if len(plainKey) != len(key) {
-		return key, nil, errors.New("crypto: received invalid plaintext key size from KMS")
+		return key, nil, Errorf("crypto: received invalid plaintext key size from KMS")
 	}
 	copy(key[:], plainKey)
 	return key, sealedKey, nil
@@ -176,7 +175,7 @@ func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 		return key, err
 	}
 	if len(plainKey) != len(key) {
-		return key, errors.New("crypto: received invalid plaintext key size from KMS")
+		return key, Errorf("crypto: received invalid plaintext key size from KMS")
 	}
 	copy(key[:], plainKey)
 	return key, nil
@@ -301,7 +300,7 @@ func (c *kesClient) parseErrorResponse(resp *http.Response) error {
 	if _, err := io.Copy(&errMsg, io.LimitReader(resp.Body, limit)); err != nil {
 		return err
 	}
-	return fmt.Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
+	return Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
 }
 
 // loadCACertificates returns a new CertPool
@@ -334,7 +333,7 @@ func loadCACertificates(path string) (*x509.CertPool, error) {
 		if os.IsNotExist(err) || os.IsPermission(err) {
 			return rootCAs, nil
 		}
-		return nil, fmt.Errorf("crypto: cannot open '%s': %v", path, err)
+		return nil, Errorf("crypto: cannot open '%s': %v", path, err)
 	}
 
 	// If path is a file, parse as PEM-encoded certifcate
@@ -346,7 +345,7 @@ func loadCACertificates(path string) (*x509.CertPool, error) {
 			return nil, err
 		}
 		if !rootCAs.AppendCertsFromPEM(cert) {
-			return nil, fmt.Errorf("crypto: '%s' is not a valid PEM-encoded certificate", path)
+			return nil, Errorf("crypto: '%s' is not a valid PEM-encoded certificate", path)
 		}
 		return rootCAs, nil
 	}

cmd/crypto/key.go

@@ -21,7 +21,6 @@ import (
 	"crypto/rand"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"io"
 	"path"
 
@@ -108,7 +107,7 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 	)
 	switch sealedKey.Algorithm {
 	default:
-		return Error(fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm))
+		return Errorf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm)
 	case SealAlgorithm:
 		mac := hmac.New(sha256.New, extKey[:])
 		mac.Write(sealedKey.IV[:])

cmd/crypto/legacy.go

@@ -17,7 +17,6 @@
 package crypto
 
 import (
-	"fmt"
 	"reflect"
 	"strconv"
 
@@ -167,7 +166,7 @@ func lookupConfigLegacy(kvs config.KVS) (VaultConfig, error) {
 	if keyVersion := env.Get(EnvLegacyVaultKeyVersion, ""); keyVersion != "" {
 		vcfg.Key.Version, err = strconv.Atoi(keyVersion)
 		if err != nil {
-			return vcfg, fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)",
+			return vcfg, Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)",
 				EnvLegacyVaultKeyVersion, keyVersion)
 		}
 	}

cmd/crypto/metadata.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/base64"
 	"errors"
-	"fmt"
 
 	"github.com/minio/minio/cmd/logger"
 )
@@ -126,7 +125,7 @@ func CreateMultipartMetadata(metadata map[string]string) map[string]string {
 // is nil.
 func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
 	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
 	}
 
 	// There are two possibilites:
@@ -172,7 +171,7 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 	b64SealedKey, ok := metadata[S3SealedKey]
 	if !ok {
-		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal sealed key for SSE-S3")
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
 	}
 
 	// There are two possibilites:
@@ -182,10 +181,10 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	keyID, idPresent := metadata[S3KMSKeyID]
 	b64KMSSealedKey, kmsKeyPresent := metadata[S3KMSSealedKey]
 	if !idPresent && kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal KMS key-ID for SSE-S3")
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
 	}
 	if idPresent && !kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
 	}
 
 	// Check whether all extracted values are well-formed
@@ -198,12 +197,12 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return keyID, kmsKey, sealedKey, Error("The internal sealed key for SSE-S3 is invalid")
+		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
 	}
 	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
 		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
 		if err != nil {
-			return keyID, kmsKey, sealedKey, Error("The internal sealed KMS data key for SSE-S3 is invalid")
+			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
 		}
 	}
 
@@ -217,7 +216,7 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 // It allocates a new metadata map if metadata is nil.
 func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
 	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
 	}
 
 	if metadata == nil {
@@ -244,7 +243,7 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	}
 	b64SealedKey, ok := metadata[SSECSealedKey]
 	if !ok {
-		return sealedKey, Error("The object metadata is missing the internal sealed key for SSE-C")
+		return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
 	}
 
 	// Check whether all extracted values are well-formed
@@ -257,7 +256,7 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return sealedKey, Error("The internal sealed key for SSE-C is invalid")
+		return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
 	}
 
 	sealedKey.Algorithm = algorithm

cmd/crypto/metadata_test.go

@@ -125,15 +125,15 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 1
 	{
-		ExpectedErr: Error("The object metadata is missing the internal sealed key for SSE-S3"),
+		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 2
 	{
-		ExpectedErr: Error("The object metadata is missing the internal KMS key-ID for SSE-S3"),
+		ExpectedErr: Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSSealedKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 3
 	{
-		ExpectedErr: Error("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
+		ExpectedErr: Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 4
@@ -150,7 +150,7 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 6
 	{
-		ExpectedErr: Error("The internal sealed key for SSE-S3 is invalid"),
+		ExpectedErr: Errorf("The internal sealed key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
 			S3KMSKeyID: "", S3KMSSealedKey: "",
@@ -158,7 +158,7 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 7
 	{
-		ExpectedErr: Error("The internal sealed KMS data key for SSE-S3 is invalid"),
+		ExpectedErr: Errorf("The internal sealed KMS data key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
 			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
@@ -188,9 +188,17 @@ var s3ParseMetadataTests = []struct {
 func TestS3ParseMetadata(t *testing.T) {
 	for i, test := range s3ParseMetadataTests {
 		keyID, dataKey, sealedKey, err := S3.ParseMetadata(test.Metadata)
-		if err != test.ExpectedErr {
+		if err != nil && test.ExpectedErr == nil {
 			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
 		}
+		if err == nil && test.ExpectedErr != nil {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if err != nil && test.ExpectedErr != nil {
+			if err.Error() != test.ExpectedErr.Error() {
+				t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+			}
+		}
 		if !bytes.Equal(dataKey, test.DataKey) {
 			t.Errorf("Test %d: got data key '%v' - want data key '%v'", i, dataKey, test.DataKey)
 		}
@@ -218,7 +226,7 @@ var ssecParseMetadataTests = []struct {
 	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
 	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
 	{
-		ExpectedErr: Error("The object metadata is missing the internal sealed key for SSE-C"),
+		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-C"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
 	}, // 2
 	{
@@ -233,7 +241,7 @@ var ssecParseMetadataTests = []struct {
 		SealedKey: SealedKey{},
 	}, // 4
 	{
-		ExpectedErr: Error("The internal sealed key for SSE-C is invalid"),
+		ExpectedErr: Errorf("The internal sealed key for SSE-C is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
 		},
@@ -267,9 +275,17 @@ func TestCreateMultipartMetadata(t *testing.T) {
 func TestSSECParseMetadata(t *testing.T) {
 	for i, test := range ssecParseMetadataTests {
 		sealedKey, err := SSEC.ParseMetadata(test.Metadata)
-		if err != test.ExpectedErr {
+		if err != nil && test.ExpectedErr == nil {
 			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
 		}
+		if err == nil && test.ExpectedErr != nil {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if err != nil && test.ExpectedErr != nil {
+			if err.Error() != test.ExpectedErr.Error() {
+				t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+			}
+		}
 		if sealedKey.Algorithm != test.SealedKey.Algorithm {
 			t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
 		}

cmd/crypto/parse.go

@@ -16,7 +16,6 @@ package crypto
 
 import (
 	"encoding/hex"
-	"fmt"
 	"strings"
 )
 
@@ -25,18 +24,18 @@ import (
 func ParseMasterKey(envArg string) (KMS, error) {
 	values := strings.SplitN(envArg, ":", 2)
 	if len(values) != 2 {
-		return nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
+		return nil, Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
 	}
 	var (
 		keyID  = values[0]
 		hexKey = values[1]
 	)
 	if len(hexKey) != 64 { // 2 hex bytes = 1 byte
-		return nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
+		return nil, Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
 	}
 	var masterKey [32]byte
 	if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
-		return nil, err
+		return nil, Errorf("Invalid KMS master key: %v", err)
 	}
 	return NewMasterKey(keyID, masterKey), nil
 }

cmd/crypto/vault.go

@@ -17,7 +17,6 @@ package crypto
 import (
 	"bytes"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -27,7 +26,7 @@ import (
 
 var (
 	//ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
-	ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
+	ErrKMSAuthLogin = Errorf("Vault service did not return auth info")
 )
 
 // VaultKey represents vault encryption key-ring.
@@ -75,17 +74,17 @@ var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements
 func (v *VaultConfig) Verify() (err error) {
 	switch {
 	case v.Endpoint == "":
-		err = errors.New("crypto: missing hashicorp vault endpoint")
+		err = Errorf("crypto: missing hashicorp vault endpoint")
 	case strings.ToLower(v.Auth.Type) != "approle":
-		err = fmt.Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
+		err = Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
 	case v.Auth.AppRole.ID == "":
-		err = errors.New("crypto: missing hashicorp vault AppRole ID")
+		err = Errorf("crypto: missing hashicorp vault AppRole ID")
 	case v.Auth.AppRole.Secret == "":
-		err = errors.New("crypto: missing hashicorp vault AppSecret ID")
+		err = Errorf("crypto: missing hashicorp vault AppSecret ID")
 	case v.Key.Name == "":
-		err = errors.New("crypto: missing hashicorp vault key name")
+		err = Errorf("crypto: missing hashicorp vault key name")
 	case v.Key.Version < 0:
-		err = errors.New("crypto: invalid hashicorp vault key version: The key version must not be negative")
+		err = Errorf("crypto: invalid hashicorp vault key version: The key version must not be negative")
 	}
 	return
 }
@@ -107,7 +106,7 @@ func NewVault(config VaultConfig) (KMS, error) {
 	}
 	client, err := vault.NewClient(&vaultCfg)
 	if err != nil {
-		return nil, err
+		return nil, Errorf("crypto: client error %w", err)
 	}
 	if config.Namespace != "" {
 		client.SetNamespace(config.Namespace)
@@ -167,6 +166,7 @@ func (v *vaultService) authenticate() (err error) {
 	var secret *vault.Secret
 	secret, err = v.client.Logical().Write("auth/approle/login", payload)
 	if err != nil {
+		err = Errorf("crypto: client error %w", err)
 		return
 	}
 	if secret == nil {
@@ -217,14 +217,23 @@ func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
 	if err != nil {
-		return key, sealedKey, err
+		return key, sealedKey, Errorf("crypto: client error %w", err)
 	}
-	sealKey := s.Data["ciphertext"].(string)
-	plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
+	sealKey, ok := s.Data["ciphertext"].(string)
+	if !ok {
+		return key, sealedKey, Errorf("crypto: incorrect 'ciphertext' key type %v", s.Data["ciphertext"])
+	}
+
+	plainKeyB64, ok := s.Data["plaintext"].(string)
+	if !ok {
+		return key, sealedKey, Errorf("crypto: incorrect 'plaintext' key type %v", s.Data["plaintext"])
+	}
+
+	plainKey, err := base64.StdEncoding.DecodeString(plainKeyB64)
 	if err != nil {
-		return key, sealedKey, err
+		return key, sealedKey, Errorf("crypto: invalid base64 key %w", err)
 	}
-	copy(key[:], []byte(plainKey))
+	copy(key[:], plainKey)
 	return key, []byte(sealKey), nil
 }
 
@@ -243,16 +252,22 @@ func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 		"ciphertext": string(sealedKey),
 		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
 	}
+
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
 	if err != nil {
-		return key, err
+		return key, Errorf("crypto: client error %w", err)
 	}
-	base64Key := s.Data["plaintext"].(string)
+
+	base64Key, ok := s.Data["plaintext"].(string)
+	if !ok {
+		return key, Errorf("crypto: incorrect 'plaintext' key type %v", s.Data["plaintext"])
+	}
+
 	plainKey, err := base64.StdEncoding.DecodeString(base64Key)
 	if err != nil {
-		return key, err
+		return key, Errorf("crypto: invalid base64 key %w", err)
 	}
-	copy(key[:], []byte(plainKey))
+	copy(key[:], plainKey)
 	return key, nil
 }
 
@@ -273,7 +288,7 @@ func (v *vaultService) UpdateKey(keyID string, sealedKey []byte, ctx Context) (r
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/rewrap/%s", keyID), payload)
 	if err != nil {
-		return nil, err
+		return nil, Errorf("crypto: client error %w", err)
 	}
 	ciphertext, ok := s.Data["ciphertext"]
 	if !ok {

cmd/daily-lifecycle-ops.go

@@ -21,8 +21,8 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/lifecycle"
 )
 
 const (
@@ -30,89 +30,31 @@ const (
 	bgLifecycleTick     = time.Hour
 )
 
-type lifecycleOps struct {
-	LastActivity time.Time
-}
-
-// Register to the daily objects listing
-var globalLifecycleOps = &lifecycleOps{}
-
-func getLocalBgLifecycleOpsStatus() BgLifecycleOpsStatus {
-	return BgLifecycleOpsStatus{
-		LastActivity: globalLifecycleOps.LastActivity,
-	}
-}
-
 // initDailyLifecycle starts the routine that receives the daily
 // listing of all objects and applies any matching bucket lifecycle
 // rules.
-func initDailyLifecycle() {
-	go startDailyLifecycle()
+func initDailyLifecycle(ctx context.Context, objAPI ObjectLayer) {
+	go startDailyLifecycle(ctx, objAPI)
 }
 
-func startDailyLifecycle() {
-	var objAPI ObjectLayer
-	var ctx = context.Background()
-
-	// Wait until the object API is ready
+func startDailyLifecycle(ctx context.Context, objAPI ObjectLayer) {
 	for {
-		objAPI = newObjectLayerWithoutSafeModeFn()
-		if objAPI == nil {
-			time.Sleep(time.Second)
-			continue
-		}
-		break
-	}
-
-	// Calculate the time of the last lifecycle operation in all peers node of the cluster
-	computeLastLifecycleActivity := func(status []BgOpsStatus) time.Time {
-		var lastAct time.Time
-		for _, st := range status {
-			if st.LifecycleOps.LastActivity.After(lastAct) {
-				lastAct = st.LifecycleOps.LastActivity
-			}
-		}
-		return lastAct
-	}
-
-	for {
-		// Check if we should perform lifecycle ops based on the last lifecycle activity, sleep one hour otherwise
-		allLifecycleStatus := []BgOpsStatus{
-			{LifecycleOps: getLocalBgLifecycleOpsStatus()},
-		}
-		if globalIsDistXL {
-			allLifecycleStatus = append(allLifecycleStatus, globalNotificationSys.BackgroundOpsStatus()...)
-		}
-		lastAct := computeLastLifecycleActivity(allLifecycleStatus)
-		if !lastAct.IsZero() && time.Since(lastAct) < bgLifecycleInterval {
-			time.Sleep(bgLifecycleTick)
-		}
-
-		// Perform one lifecycle operation
-		err := lifecycleRound(ctx, objAPI)
-		switch err.(type) {
-		// Unable to hold a lock means there is another
-		// instance doing the lifecycle round round
-		case OperationTimedOut:
-			time.Sleep(bgLifecycleTick)
-		default:
-			logger.LogIf(ctx, err)
-			time.Sleep(time.Minute)
-			continue
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.NewTimer(bgLifecycleInterval).C:
+			// Perform one lifecycle operation
+			logger.LogIf(ctx, lifecycleRound(ctx, objAPI))
 		}
 
 	}
 }
 
-var lifecycleTimeout = newDynamicTimeout(60*time.Second, time.Second)
-
 func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
-	// Lock to avoid concurrent lifecycle ops from other nodes
-	sweepLock := objAPI.NewNSLock(ctx, "system", "daily-lifecycle-ops")
-	if err := sweepLock.GetLock(lifecycleTimeout); err != nil {
-		return err
+	// No action is expected when WORM is enabled
+	if globalWORMEnabled {
+		return nil
 	}
-	defer sweepLock.Unlock()
 
 	buckets, err := objAPI.ListBuckets(ctx)
 	if err != nil {
@@ -126,32 +68,46 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 			continue
 		}
 
+		_, bucketHasLockConfig := globalBucketObjectLockConfig.Get(bucket.Name)
+
 		// Calculate the common prefix of all lifecycle rules
 		var prefixes []string
 		for _, rule := range l.Rules {
-			prefixes = append(prefixes, rule.Filter.Prefix)
+			prefixes = append(prefixes, rule.Prefix())
 		}
 		commonPrefix := lcp(prefixes)
 
-		// List all objects and calculate lifecycle action based on object name & object modtime
-		marker := ""
+		// Allocate new results channel to receive ObjectInfo.
+		objInfoCh := make(chan ObjectInfo)
+
+		// Walk through all objects
+		if err := objAPI.Walk(ctx, bucket.Name, commonPrefix, objInfoCh); err != nil {
+			return err
+		}
+
 		for {
-			res, err := objAPI.ListObjects(ctx, bucket.Name, commonPrefix, marker, "", maxObjectList)
-			if err != nil {
-				continue
-			}
 			var objects []string
-			for _, obj := range res.Objects {
+			for obj := range objInfoCh {
+				if len(objects) == maxObjectList {
+					// Reached maximum delete requests, attempt a delete for now.
+					break
+				}
 				// Find the action that need to be executed
-				action := l.ComputeAction(obj.Name, obj.ModTime)
-				switch action {
-				case lifecycle.DeleteAction:
+				if l.ComputeAction(obj.Name, obj.UserTags, obj.ModTime) == lifecycle.DeleteAction {
+					if bucketHasLockConfig && enforceRetentionForLifecycle(ctx, obj) {
+						continue
+					}
 					objects = append(objects, obj.Name)
-				default:
-					// Do nothing, for now.
 				}
 			}
 
+			// Nothing to do.
+			if len(objects) == 0 {
+				break
+			}
+
+			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()))
+
 			// Deletes a list of objects.
 			deleteErrs, err := objAPI.DeleteObjects(ctx, bucket.Name, objects)
 			if err != nil {
@@ -173,12 +129,6 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 					})
 				}
 			}
-
-			if !res.IsTruncated {
-				// We are done here, proceed to next bucket.
-				break
-			}
-			marker = res.NextMarker
 		}
 	}
 

cmd/data-usage-cache.go

@@ -0,0 +1,517 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/cespare/xxhash/v2"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/tinylib/msgp/msgp"
+)
+
+const dataUsageHashLen = 8
+
+//go:generate msgp -file $GOFILE -unexported
+
+// dataUsageHash is the hash type used.
+type dataUsageHash uint64
+
+// sizeHistogram is a size histogram.
+type sizeHistogram [dataUsageBucketLen]uint64
+
+//msgp:tuple dataUsageEntry
+type dataUsageEntry struct {
+	// These fields do no include any children.
+	Size     int64
+	Objects  uint64
+	ObjSizes sizeHistogram
+
+	Children dataUsageHashMap
+}
+
+//msgp:ignore dataUsageEntryInfo
+type dataUsageEntryInfo struct {
+	Name   string
+	Parent string
+	Entry  dataUsageEntry
+}
+
+type dataUsageCacheInfo struct {
+	// Name of the bucket. Also root element.
+	Name       string
+	LastUpdate time.Time
+	NextCycle  uint8
+}
+
+// merge other data usage entry into this, excluding children.
+func (e *dataUsageEntry) merge(other dataUsageEntry) {
+	e.Objects += other.Objects
+	e.Size += other.Size
+	for i, v := range other.ObjSizes[:] {
+		e.ObjSizes[i] += v
+	}
+}
+
+// mod returns true if the hash mod cycles == cycle.
+func (h dataUsageHash) mod(cycle uint8, cycles uint8) bool {
+	return uint8(h)%cycles == cycle%cycles
+}
+
+// addChildString will add a child based on its name.
+// If it already exists it will not be added again.
+func (e *dataUsageEntry) addChildString(name string) {
+	e.addChild(hashPath(name))
+}
+
+// addChild will add a child based on its hash.
+// If it already exists it will not be added again.
+func (e *dataUsageEntry) addChild(hash dataUsageHash) {
+	if _, ok := e.Children[hash]; ok {
+		return
+	}
+	if e.Children == nil {
+		e.Children = make(dataUsageHashMap, 1)
+	}
+	e.Children[hash] = struct{}{}
+}
+
+// find a path in the cache.
+// Returns nil if not found.
+func (d *dataUsageCache) find(path string) *dataUsageEntry {
+	due, ok := d.Cache[hashPath(path)]
+	if !ok {
+		return nil
+	}
+	return &due
+}
+
+// dui converts the flattened version of the path to DataUsageInfo.
+func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
+	e := d.find(path)
+	if e == nil {
+		return DataUsageInfo{LastUpdate: UTCNow()}
+	}
+	flat := d.flatten(*e)
+	return DataUsageInfo{
+		LastUpdate:            d.Info.LastUpdate,
+		ObjectsCount:          flat.Objects,
+		ObjectsTotalSize:      uint64(flat.Size),
+		ObjectsSizesHistogram: flat.ObjSizes.asMap(),
+		BucketsCount:          uint64(len(e.Children)),
+		BucketsSizes:          d.pathSizes(buckets),
+	}
+}
+
+// replace will add or replace an entry in the cache.
+// If a parent is specified it will be added to that if not already there.
+// If the parent does not exist, it will be added.
+func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) {
+	hash := hashPath(path)
+	if d.Cache == nil {
+		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+	}
+	d.Cache[hash] = e
+	if parent != "" {
+		phash := hashPath(parent)
+		p := d.Cache[phash]
+		p.addChild(hash)
+		d.Cache[phash] = p
+	}
+}
+
+// replaceHashed add or replaces an entry to the cache based on its hash.
+// If a parent is specified it will be added to that if not already there.
+// If the parent does not exist, it will be added.
+func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash, e dataUsageEntry) {
+	if d.Cache == nil {
+		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+	}
+	d.Cache[hash] = e
+	if parent != nil {
+		p := d.Cache[*parent]
+		p.addChild(hash)
+		d.Cache[*parent] = p
+	}
+}
+
+// StringAll returns a detailed string representation of all entries in the cache.
+func (d *dataUsageCache) StringAll() string {
+	s := fmt.Sprintf("info:%+v\n", d.Info)
+	for k, v := range d.Cache {
+		s += fmt.Sprintf("\t%v: %+v\n", k, v)
+	}
+	return strings.TrimSpace(s)
+}
+
+// String returns a human readable representation of the string.
+func (h dataUsageHash) String() string {
+	return fmt.Sprintf("%x", uint64(h))
+}
+
+// flatten all children of the root into the root element and return it.
+func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
+	for id := range root.Children {
+		e := d.Cache[id]
+		if len(e.Children) > 0 {
+			e = d.flatten(e)
+		}
+		root.merge(e)
+	}
+	root.Children = nil
+	return root
+}
+
+// add a size to the histogram.
+func (h *sizeHistogram) add(size int64) {
+	// Fetch the histogram interval corresponding
+	// to the passed object size.
+	for i, interval := range ObjectsHistogramIntervals {
+		if size >= interval.start && size <= interval.end {
+			h[i]++
+			break
+		}
+	}
+}
+
+// asMap returns the map as a map[string]uint64.
+func (h *sizeHistogram) asMap() map[string]uint64 {
+	res := make(map[string]uint64, 7)
+	for i, count := range h {
+		res[ObjectsHistogramIntervals[i].name] = count
+	}
+	return res
+}
+
+// pathSizes returns the path sizes as a map.
+func (d *dataUsageCache) pathSizes(buckets []BucketInfo) map[string]uint64 {
+	var dst = make(map[string]uint64, len(buckets))
+	for _, bucket := range buckets {
+		e := d.find(bucket.Name)
+		if e == nil {
+			continue
+		}
+		flat := d.flatten(*e)
+		dst[bucket.Name] = uint64(flat.Size)
+	}
+	return dst
+}
+
+// sizeRecursive returns the path as a flattened entry.
+func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
+	root := d.find(path)
+	if root == nil || len(root.Children) == 0 {
+		return root
+	}
+	flat := d.flatten(*root)
+	return &flat
+}
+
+// dataUsageCache contains a cache of data usage entries.
+//msgp:ignore dataUsageCache
+type dataUsageCache struct {
+	Info  dataUsageCacheInfo
+	Cache map[dataUsageHash]dataUsageEntry
+}
+
+// root returns the root of the cache.
+func (d *dataUsageCache) root() *dataUsageEntry {
+	return d.find(d.Info.Name)
+}
+
+// rootHash returns the root of the cache.
+func (d *dataUsageCache) rootHash() dataUsageHash {
+	return hashPath(d.Info.Name)
+}
+
+// clone returns a copy of the cache with no references to the existing.
+func (d *dataUsageCache) clone() dataUsageCache {
+	clone := dataUsageCache{
+		Info:  d.Info,
+		Cache: make(map[dataUsageHash]dataUsageEntry, len(d.Cache)),
+	}
+	for k, v := range d.Cache {
+		clone.Cache[k] = v
+	}
+	return clone
+}
+
+// merge root of other into d.
+// children of root will be flattened before being merged.
+// Last update time will be set to the last updated.
+func (d *dataUsageCache) merge(other dataUsageCache) {
+	existingRoot := d.root()
+	otherRoot := other.root()
+	if existingRoot == nil && otherRoot == nil {
+		return
+	}
+	if otherRoot == nil {
+		return
+	}
+	if existingRoot == nil {
+		*d = other.clone()
+		return
+	}
+	if other.Info.LastUpdate.After(d.Info.LastUpdate) {
+		d.Info.LastUpdate = other.Info.LastUpdate
+	}
+	existingRoot.merge(*otherRoot)
+	eHash := d.rootHash()
+	for key := range otherRoot.Children {
+		entry := other.Cache[key]
+		flat := other.flatten(entry)
+		existing := d.Cache[key]
+		// If not found, merging simply adds.
+		existing.merge(flat)
+		d.replaceHashed(key, &eHash, existing)
+	}
+}
+
+// load the cache content with name from minioMetaBackgroundOpsBucket.
+// Only backend errors are returned as errors.
+// If the object is not found or unable to deserialize d is cleared and nil error is returned.
+func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name string) error {
+	var buf bytes.Buffer
+	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
+	if err != nil {
+		if !isErrObjectNotFound(err) {
+			return toObjectErr(err, dataUsageBucket, name)
+		}
+		*d = dataUsageCache{}
+		return nil
+	}
+	err = d.deserialize(buf.Bytes())
+	if err != nil {
+		*d = dataUsageCache{}
+		logger.LogIf(ctx, err)
+	}
+	return nil
+}
+
+// save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
+func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name string) error {
+	b := d.serialize()
+	size := int64(len(b))
+	r, err := hash.NewReader(bytes.NewReader(b), size, "", "", size, false)
+	if err != nil {
+		return err
+	}
+
+	_, err = store.PutObject(ctx,
+		dataUsageBucket,
+		name,
+		NewPutObjReader(r, nil, nil),
+		ObjectOptions{})
+	return err
+}
+
+// dataUsageCacheVer indicates the cache version.
+// Bumping the cache version will drop data from previous versions
+// and write new data with the new version.
+const dataUsageCacheVer = 1
+
+// serialize the contents of the cache.
+func (d *dataUsageCache) serialize() []byte {
+	// Alloc pessimistically
+	// dataUsageCacheVer
+	due := dataUsageEntry{}
+	msgLen := 1
+	msgLen += d.Info.Msgsize()
+	// len(d.Cache)
+	msgLen += binary.MaxVarintLen64
+	// Hashes (one for key, assume 1 child/node)
+	msgLen += len(d.Cache) * dataUsageHashLen * 2
+	msgLen += len(d.Cache) * due.Msgsize()
+
+	// Create destination buffer...
+	dst := make([]byte, 0, msgLen)
+
+	var n int
+	tmp := make([]byte, 1024)
+	// byte: version.
+	dst = append(dst, dataUsageCacheVer)
+	// Info...
+	dst, err := d.Info.MarshalMsg(dst)
+	if err != nil {
+		panic(err)
+	}
+	n = binary.PutUvarint(tmp, uint64(len(d.Cache)))
+	dst = append(dst, tmp[:n]...)
+
+	for k, v := range d.Cache {
+		// Put key
+		binary.LittleEndian.PutUint64(tmp[:dataUsageHashLen], uint64(k))
+		dst = append(dst, tmp[:8]...)
+		tmp, err = v.MarshalMsg(tmp[:0])
+		if err != nil {
+			panic(err)
+		}
+		// key, value pairs.
+		dst = append(dst, tmp...)
+
+	}
+	return dst
+}
+
+// deserialize the supplied byte slice into the cache.
+func (d *dataUsageCache) deserialize(b []byte) error {
+	if len(b) < 1 {
+		return io.ErrUnexpectedEOF
+	}
+	switch b[0] {
+	case 1:
+	default:
+		return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
+	}
+	b = b[1:]
+
+	// Info...
+	b, err := d.Info.UnmarshalMsg(b)
+	if err != nil {
+		return err
+	}
+	cacheLen, n := binary.Uvarint(b)
+	if n <= 0 {
+		return fmt.Errorf("dataUsageCache: reading cachelen, n <= 0 ")
+	}
+	b = b[n:]
+	d.Cache = make(map[dataUsageHash]dataUsageEntry, cacheLen)
+
+	for i := 0; i < int(cacheLen); i++ {
+		if len(b) <= dataUsageHashLen {
+			return io.ErrUnexpectedEOF
+		}
+		k := binary.LittleEndian.Uint64(b[:dataUsageHashLen])
+		b = b[dataUsageHashLen:]
+		var v dataUsageEntry
+		b, err = v.UnmarshalMsg(b)
+		if err != nil {
+			return err
+		}
+		d.Cache[dataUsageHash(k)] = v
+	}
+	return nil
+}
+
+// Trim this from start+end of hashes.
+var hashPathCutSet = dataUsageRoot
+
+func init() {
+	if dataUsageRoot != string(filepath.Separator) {
+		hashPathCutSet = dataUsageRoot + string(filepath.Separator)
+	}
+}
+
+// hashPath calculates a hash of the provided string.
+func hashPath(data string) dataUsageHash {
+	if data != dataUsageRoot {
+		data = strings.Trim(data, hashPathCutSet)
+	}
+	data = path.Clean(data)
+	return dataUsageHash(xxhash.Sum64String(data))
+}
+
+//msgp:ignore dataUsageEntryInfo
+type dataUsageHashMap map[dataUsageHash]struct{}
+
+// MarshalMsg implements msgp.Marshaler
+func (d dataUsageHashMap) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, d.Msgsize())
+
+	// Write bin header manually
+	const mbin32 uint8 = 0xc6
+	sz := uint32(len(d)) * dataUsageHashLen
+	o = append(o, mbin32, byte(sz>>24), byte(sz>>16), byte(sz>>8), byte(sz))
+
+	var tmp [dataUsageHashLen]byte
+	for k := range d {
+		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
+		o = append(o, tmp[:]...)
+	}
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (d dataUsageHashMap) Msgsize() (s int) {
+	s = 5 + len(d)*dataUsageHashLen
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (d *dataUsageHashMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var hashes []byte
+	hashes, bts, err = msgp.ReadBytesZC(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "dataUsageHashMap")
+		return
+	}
+
+	var dst = make(dataUsageHashMap, len(hashes)/dataUsageHashLen)
+	for len(hashes) >= dataUsageHashLen {
+		dst[dataUsageHash(binary.LittleEndian.Uint64(hashes[:dataUsageHashLen]))] = struct{}{}
+		hashes = hashes[dataUsageHashLen:]
+	}
+	*d = dst
+	o = bts
+	return
+}
+
+func (d *dataUsageHashMap) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadBytesHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	var dst = make(dataUsageHashMap, zb0001)
+	var tmp [8]byte
+	for i := uint32(0); i < zb0001; i++ {
+		_, err = io.ReadFull(dc, tmp[:])
+		if err != nil {
+			err = msgp.WrapError(err, "dataUsageHashMap")
+			return
+		}
+		dst[dataUsageHash(binary.LittleEndian.Uint64(tmp[:]))] = struct{}{}
+	}
+	return nil
+}
+func (d dataUsageHashMap) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteBytesHeader(uint32(len(d)) * dataUsageHashLen)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	var tmp [dataUsageHashLen]byte
+	for k := range d {
+		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
+		_, err = en.Write(tmp[:])
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+	}
+	return nil
+}

cmd/data-usage-cache_gen.go

@@ -0,0 +1,447 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "LastUpdate":
+			z.LastUpdate, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "LastUpdate")
+				return
+			}
+		case "NextCycle":
+			z.NextCycle, err = dc.ReadUint8()
+			if err != nil {
+				err = msgp.WrapError(err, "NextCycle")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 3
+	// write "Name"
+	err = en.Append(0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Name)
+	if err != nil {
+		err = msgp.WrapError(err, "Name")
+		return
+	}
+	// write "LastUpdate"
+	err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.LastUpdate)
+	if err != nil {
+		err = msgp.WrapError(err, "LastUpdate")
+		return
+	}
+	// write "NextCycle"
+	err = en.Append(0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint8(z.NextCycle)
+	if err != nil {
+		err = msgp.WrapError(err, "NextCycle")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 3
+	// string "Name"
+	o = append(o, 0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.Name)
+	// string "LastUpdate"
+	o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
+	o = msgp.AppendTime(o, z.LastUpdate)
+	// string "NextCycle"
+	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
+	o = msgp.AppendUint8(o, z.NextCycle)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "LastUpdate":
+			z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LastUpdate")
+				return
+			}
+		case "NextCycle":
+			z.NextCycle, bts, err = msgp.ReadUint8Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NextCycle")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z dataUsageCacheInfo) Msgsize() (s int) {
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint8Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	z.Size, err = dc.ReadInt64()
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.Objects, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.DecodeMsg(dc)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
+	// array header, size 4
+	err = en.Append(0x94)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.Size)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	err = en.WriteUint64(z.Objects)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		err = en.WriteUint64(z.ObjSizes[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// array header, size 4
+	o = append(o, 0x94)
+	o = msgp.AppendInt64(o, z.Size)
+	o = msgp.AppendUint64(o, z.Objects)
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
+	for za0001 := range z.ObjSizes {
+		o = msgp.AppendUint64(o, z.ObjSizes[za0001])
+	}
+	o, err = z.Children.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	z.Size, bts, err = msgp.ReadInt64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	bts, err = z.Children.UnmarshalMsg(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageEntry) Msgsize() (s int) {
+	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
+	{
+		var zb0001 uint64
+		zb0001, err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = dataUsageHash(zb0001)
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z dataUsageHash) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteUint64(uint64(z))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z dataUsageHash) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendUint64(o, uint64(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageHash) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 uint64
+		zb0001, bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = dataUsageHash(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z dataUsageHash) Msgsize() (s int) {
+	s = msgp.Uint64Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *sizeHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *sizeHistogram) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for za0001 := range z {
+		err = en.WriteUint64(z[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *sizeHistogram) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
+	for za0001 := range z {
+		o = msgp.AppendUint64(o, z[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *sizeHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *sizeHistogram) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size))
+	return
+}

cmd/data-usage-cache_gen_test.go

@@ -0,0 +1,349 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshaldataUsageCacheInfo(t *testing.T) {
+	v := dataUsageCacheInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageCacheInfo(t *testing.T) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageCacheInfo Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageCacheInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshaldataUsageEntry(t *testing.T) {
+	v := dataUsageEntry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageEntry(t *testing.T) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageEntry Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageEntry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalsizeHistogram(t *testing.T) {
+	v := sizeHistogram{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodesizeHistogram(t *testing.T) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodesizeHistogram Msgsize() is inaccurate")
+	}
+
+	vn := sizeHistogram{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodesizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodesizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/data-usage.go

@@ -20,119 +20,90 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
+	"os"
+	"path"
+	"strconv"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/hash"
 )
 
 const (
-	dataUsageObjName       = "data-usage"
-	dataUsageCrawlInterval = 12 * time.Hour
+	dataUsageObjName         = ".usage.json"
+	dataUsageCacheName       = ".usage-cache.bin"
+	envDataUsageCrawlConf    = "MINIO_DISK_USAGE_CRAWL_ENABLE"
+	envDataUsageCrawlDelay   = "MINIO_DISK_USAGE_CRAWL_DELAY"
+	envDataUsageCrawlDebug   = "MINIO_DISK_USAGE_CRAWL_DEBUG"
+	dataUsageSleepPerFolder  = 1 * time.Millisecond
+	dataUsageSleepDefMult    = 10.0
+	dataUsageUpdateDirCycles = 16
+	dataUsageRoot            = SlashSeparator
+	dataUsageBucket          = minioMetaBucket + SlashSeparator + bucketMetaPrefix
+	dataUsageStartDelay      = 5 * time.Minute // Time to wait on startup and between cycles.
 )
 
-func initDataUsageStats() {
-	go runDataUsageInfoUpdateRoutine()
-}
-
-func runDataUsageInfoUpdateRoutine() {
-	// Wait until the object layer is ready
-	var objAPI ObjectLayer
-	for {
-		objAPI = newObjectLayerWithoutSafeModeFn()
-		if objAPI == nil {
-			time.Sleep(time.Second)
-			continue
-		}
-		break
-	}
-
-	ctx := context.Background()
-
-	switch v := objAPI.(type) {
-	case *xlZones:
-		runDataUsageInfoForXLZones(ctx, v, GlobalServiceDoneCh)
-	case *FSObjects:
-		runDataUsageInfoForFS(ctx, v, GlobalServiceDoneCh)
-	default:
-		return
+// initDataUsageStats will start the crawler unless disabled.
+func initDataUsageStats(ctx context.Context, objAPI ObjectLayer) {
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
+		go runDataUsageInfo(ctx, objAPI)
 	}
 }
 
-func runDataUsageInfoForFS(ctx context.Context, fsObj *FSObjects, endCh <-chan struct{}) {
-	t := time.NewTicker(dataUsageCrawlInterval)
-	defer t.Stop()
+func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer) {
 	for {
-		// Get data usage info of the FS Object
-		usageInfo := fsObj.crawlAndGetDataUsageInfo(ctx, endCh)
-		// Save the data usage in the disk
-		err := storeDataUsageInBackend(ctx, fsObj, usageInfo)
-		if err != nil {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.NewTimer(dataUsageStartDelay).C:
+			// Wait before starting next cycle and wait on startup.
+			results := make(chan DataUsageInfo, 1)
+			go storeDataUsageInBackend(ctx, objAPI, results)
+			err := objAPI.CrawlAndGetDataUsage(ctx, results)
+			close(results)
 			logger.LogIf(ctx, err)
 		}
-		select {
-		case <-endCh:
-			return
-		// Wait until the next crawl interval
-		case <-t.C:
-		}
 	}
 }
 
-func runDataUsageInfoForXLZones(ctx context.Context, z *xlZones, endCh <-chan struct{}) {
-	locker := z.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
-	for {
-		err := locker.GetLock(newDynamicTimeout(time.Millisecond, time.Millisecond))
-		if err != nil {
-			time.Sleep(5 * time.Minute)
-			continue
-		}
-		// Break without locking
-		break
-	}
-
-	t := time.NewTicker(dataUsageCrawlInterval)
-	defer t.Stop()
-	for {
-		usageInfo := z.crawlAndGetDataUsage(ctx, endCh)
-		err := storeDataUsageInBackend(ctx, z, usageInfo)
+// storeDataUsageInBackend will store all objects sent on the gui channel until closed.
+func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan DataUsageInfo) {
+	for dataUsageInfo := range gui {
+		dataUsageJSON, err := json.Marshal(dataUsageInfo)
 		if err != nil {
 			logger.LogIf(ctx, err)
+			continue
 		}
-		select {
-		case <-endCh:
-			locker.Unlock()
-			return
-		case <-t.C:
+		size := int64(len(dataUsageJSON))
+		r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
 		}
-	}
-}
 
-func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dataUsageInfo DataUsageInfo) error {
-	dataUsageJSON, err := json.Marshal(dataUsageInfo)
-	if err != nil {
-		return err
+		_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
+		logger.LogIf(ctx, err)
 	}
-
-	size := int64(len(dataUsageJSON))
-	r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
-	if err != nil {
-		return err
-	}
-
-	_, err = objAPI.PutObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
-	return err
 }
 
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	var dataUsageInfoJSON bytes.Buffer
 
-	err := objAPI.GetObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
+	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
 	if err != nil {
-		return DataUsageInfo{}, nil
+		if isErrObjectNotFound(err) {
+			return DataUsageInfo{}, nil
+		}
+		return DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
 	}
 
 	var dataUsageInfo DataUsageInfo
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	err = json.Unmarshal(dataUsageInfoJSON.Bytes(), &dataUsageInfo)
 	if err != nil {
 		return DataUsageInfo{}, err
@@ -140,3 +111,324 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 
 	return dataUsageInfo, nil
 }
+
+// Item represents each file while walking.
+type Item struct {
+	Path string
+	Typ  os.FileMode
+}
+
+type getSizeFn func(item Item) (int64, error)
+
+type cachedFolder struct {
+	name   string
+	parent *dataUsageHash
+}
+
+type folderScanner struct {
+	root               string
+	getSize            getSizeFn
+	oldCache           dataUsageCache
+	newCache           dataUsageCache
+	waitForLowActiveIO func()
+
+	dataUsageCrawlMult  float64
+	dataUsageCrawlDebug bool
+
+	newFolders      []cachedFolder
+	existingFolders []cachedFolder
+}
+
+// sleepDuration multiplies the duration d by x and sleeps if is more than 100 micro seconds.
+// sleep is limited to max 1 second.
+func sleepDuration(d time.Duration, x float64) {
+	// Don't sleep for really small amount of time
+	if d := time.Duration(float64(d) * x); d > time.Microsecond*100 {
+		if d > time.Second {
+			d = time.Second
+		}
+		time.Sleep(d)
+	}
+}
+
+// scanQueuedLevels will scan the provided folders.
+// Files found in the folders will be added to f.newCache.
+// If final is provided folders will be put into f.newFolders or f.existingFolders.
+// If final is not provided the folders found are returned from the function.
+func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
+	var nextFolders []cachedFolder
+	done := ctx.Done()
+	for _, folder := range folders {
+		select {
+		case <-done:
+			return nil, ctx.Err()
+		default:
+		}
+
+		f.waitForLowActiveIO()
+		sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
+
+		cache := dataUsageEntry{}
+		thisHash := hashPath(folder.name)
+
+		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
+			// Parse
+			entName = path.Clean(path.Join(folder.name, entName))
+			bucket, _ := path2BucketObjectWithBasePath(f.root, entName)
+			if bucket == "" {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" no bucket (%s,%s)", f.root, entName)
+				}
+				return nil
+			}
+
+			if isReservedOrInvalidBucket(bucket, false) {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" invalid bucket: %v, entry: %v", bucket, entName)
+				}
+				return nil
+			}
+
+			select {
+			case <-done:
+				return ctx.Err()
+			default:
+			}
+
+			if typ&os.ModeDir != 0 {
+				h := hashPath(entName)
+				_, exists := f.oldCache.Cache[h]
+				cache.addChildString(entName)
+
+				this := cachedFolder{name: entName, parent: &thisHash}
+				cache.addChild(h)
+				if final {
+					if exists {
+						f.existingFolders = append(f.existingFolders, this)
+					} else {
+						f.newFolders = append(f.newFolders, this)
+					}
+				} else {
+					nextFolders = append(nextFolders, this)
+				}
+				return nil
+			}
+			f.waitForLowActiveIO()
+			// Dynamic time delay.
+			t := UTCNow()
+
+			// Get file size, ignore errors.
+			size, err := f.getSize(Item{Path: path.Join(f.root, entName), Typ: typ})
+
+			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			if err == errSkipFile {
+				return nil
+			}
+			logger.LogIf(ctx, err)
+			cache.Size += size
+			cache.Objects++
+			cache.ObjSizes.add(size)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		f.newCache.replaceHashed(thisHash, folder.parent, cache)
+	}
+	return nextFolders, nil
+}
+
+// deepScanFolder will deep scan a folder and return the size if no error occurs.
+func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dataUsageEntry, error) {
+	var cache dataUsageEntry
+
+	done := ctx.Done()
+
+	var addDir func(entName string, typ os.FileMode) error
+	var dirStack = []string{f.root, folder}
+
+	addDir = func(entName string, typ os.FileMode) error {
+		select {
+		case <-done:
+			return ctx.Err()
+		default:
+		}
+
+		f.waitForLowActiveIO()
+		if typ&os.ModeDir != 0 {
+			dirStack = append(dirStack, entName)
+			err := readDirFn(path.Join(dirStack...), addDir)
+			dirStack = dirStack[:len(dirStack)-1]
+			sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
+			return err
+		}
+
+		// Dynamic time delay.
+		t := UTCNow()
+
+		// Get file size, ignore errors.
+		dirStack = append(dirStack, entName)
+		fileName := path.Join(dirStack...)
+		dirStack = dirStack[:len(dirStack)-1]
+
+		size, err := f.getSize(Item{Path: fileName, Typ: typ})
+
+		// Don't sleep for really small amount of time
+		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+
+		if err == errSkipFile {
+			return nil
+		}
+		logger.LogIf(ctx, err)
+		cache.Size += size
+		cache.Objects++
+		cache.ObjSizes.add(size)
+		return nil
+	}
+	err := readDirFn(path.Join(dirStack...), addDir)
+	if err != nil {
+		return nil, err
+	}
+	return &cache, nil
+}
+
+// updateUsage will crawl the basepath+cache.Info.Name and return an updated cache.
+// The returned cache will always be valid, but may not be updated from the existing.
+// Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
+// If the supplied context is canceled the function will return at the first chance.
+func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
+	t := UTCNow()
+
+	dataUsageDebug := env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn
+	defer func() {
+		if dataUsageDebug {
+			logger.Info(color.Green("updateUsage")+" Crawl time at %s: %v", basePath, time.Since(t))
+		}
+	}()
+
+	if cache.Info.Name == "" {
+		cache.Info.Name = dataUsageRoot
+	}
+
+	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		delayMult = dataUsageSleepDefMult
+	}
+
+	s := folderScanner{
+		root:                basePath,
+		getSize:             getSize,
+		oldCache:            cache,
+		newCache:            dataUsageCache{Info: cache.Info},
+		waitForLowActiveIO:  waitForLowActiveIO,
+		newFolders:          nil,
+		existingFolders:     nil,
+		dataUsageCrawlMult:  delayMult,
+		dataUsageCrawlDebug: dataUsageDebug,
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(color.Green("runDataUsageInfo:") + " Starting crawler master")
+	}
+
+	done := ctx.Done()
+	var flattenLevels = 3
+
+	// If we are scanning inside a bucket reduce depth by 1.
+	if cache.Info.Name != dataUsageRoot {
+		flattenLevels--
+	}
+
+	var logPrefix, logSuffix string
+	if s.dataUsageCrawlDebug {
+		logPrefix = color.Green("data-usage: ")
+		logSuffix = color.Blue(" - %v + %v", basePath, cache.Info.Name)
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Cycle: %v"+logSuffix, cache.Info.NextCycle)
+	}
+
+	// Always scan flattenLevels deep. Cache root is level 0.
+	todo := []cachedFolder{{name: cache.Info.Name}}
+	for i := 0; i < flattenLevels; i++ {
+		if s.dataUsageCrawlDebug {
+			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
+		}
+		select {
+		case <-done:
+			return cache, ctx.Err()
+		default:
+		}
+		var err error
+		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1)
+		if err != nil {
+			// No useful information...
+			return cache, err
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
+	}
+
+	// Add new folders first
+	for _, folder := range s.newFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			continue
+		}
+		s.newCache.replace(folder.name, "", *du)
+		// Add to parent manually
+		if folder.parent != nil {
+			parent := s.newCache.Cache[*folder.parent]
+			parent.addChildString(folder.name)
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
+	}
+
+	// Do selective scanning of existing folders.
+	for _, folder := range s.existingFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		h := hashPath(folder.name)
+		if !h.mod(s.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
+			s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h])
+			continue
+		}
+
+		// Update on this cycle...
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			continue
+		}
+		s.newCache.replaceHashed(h, folder.parent, *du)
+	}
+
+	s.newCache.Info.LastUpdate = UTCNow()
+	s.newCache.Info.NextCycle++
+	return s.newCache, nil
+}

cmd/data-usage_test.go

@@ -0,0 +1,651 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+type usageTestFile struct {
+	name string
+	size int
+}
+
+func TestDataUsageUpdate(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUsageUpdate")
+	if err != nil {
+		t.Skip(err)
+	}
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "rootfile", size: 10000},
+		{name: "rootfile2", size: 10000},
+		{name: "dir1/d1file", size: 2000},
+		{name: "dir2/d2file", size: 300},
+		{name: "dir1/dira/dafile", size: 100000},
+		{name: "dir1/dira/dbfile", size: 200000},
+		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+
+	got, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test dirs
+	var want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    1322310,
+			flatten: true,
+			objs:    8,
+			oSizes:  sizeHistogram{0: 2, 1: 6},
+		},
+		{
+			path:   "/",
+			size:   20000,
+			objs:   2,
+			oSizes: sizeHistogram{1: 2},
+		},
+		{
+			path:   "/dir1",
+			size:   2000,
+			objs:   1,
+			oSizes: sizeHistogram{1: 1},
+		},
+		{
+			path:    "/dir1/dira",
+			flatten: true,
+			size:    1300010,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:    "/dir1/dira/",
+			flatten: true,
+			size:    1300010,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:   "/dir1/dira",
+			size:   300000,
+			objs:   2,
+			oSizes: sizeHistogram{0: 0, 1: 2},
+		},
+		{
+			path:   "/dir1/dira/",
+			size:   300000,
+			objs:   2,
+			oSizes: sizeHistogram{0: 0, 1: 2},
+		},
+		{
+			path:  "/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "newfolder/afile",
+			size: 4,
+		},
+		{
+			name: "newfolder/anotherone",
+			size: 1,
+		},
+		{
+			name: "newfolder/anemptyone",
+			size: 0,
+		},
+		{
+			name: "dir1/fileindir1",
+			size: 20000,
+		},
+		{
+			name: "dir1/dirc/fileindirc",
+			size: 20000,
+		},
+		{
+			name: "rootfile3",
+			size: 1000,
+		},
+	}
+	createUsageTestFiles(t, base, files)
+	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    1363315,
+			flatten: true,
+			objs:    14,
+			oSizes:  sizeHistogram{0: 6, 1: 8},
+		},
+		{
+			path:   "/",
+			size:   21000,
+			objs:   3,
+			oSizes: sizeHistogram{0: 1, 1: 2},
+		},
+		{
+			path:   "/newfolder",
+			size:   5,
+			objs:   3,
+			oSizes: sizeHistogram{0: 3},
+		},
+		{
+			path:    "/dir1/dira",
+			size:    1300010,
+			flatten: true,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "dir1/dira/dirasub/fileindira2",
+			size: 200,
+		},
+	}
+
+	createUsageTestFiles(t, base, files)
+	err = os.RemoveAll(filepath.Join(base, "dir1/dira/dirasub/dcfile"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Changed dir must be picked up in this many cycles.
+	for i := 0; i < dataUsageUpdateDirCycles; i++ {
+		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    363515,
+			flatten: true,
+			objs:    14,
+			oSizes:  sizeHistogram{0: 7, 1: 7},
+		},
+		{
+			path:    "/dir1/dira",
+			size:    300210,
+			objs:    4,
+			flatten: true,
+			oSizes:  sizeHistogram{0: 2, 1: 2},
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+}
+
+func TestDataUsageUpdatePrefix(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUpdateUsagePrefix")
+	if err != nil {
+		t.Skip(err)
+	}
+	base = filepath.Join(base, "bucket")
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "bucket/rootfile", size: 10000},
+		{name: "bucket/rootfile2", size: 10000},
+		{name: "bucket/dir1/d1file", size: 2000},
+		{name: "bucket/dir2/d2file", size: 300},
+		{name: "bucket/dir1/dira/dafile", size: 100000},
+		{name: "bucket/dir1/dira/dbfile", size: 200000},
+		{name: "bucket/dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "bucket/dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+	got, err := updateUsage(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test dirs
+	var want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   1322310,
+			objs:   8,
+			oSizes: sizeHistogram{0: 2, 1: 6},
+		},
+		{
+			path:   "bucket/",
+			size:   20000,
+			objs:   2,
+			oSizes: sizeHistogram{1: 2},
+		},
+		{
+			path:   "bucket/dir1",
+			size:   2000,
+			objs:   1,
+			oSizes: sizeHistogram{1: 1},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:   "bucket/dir1/dira/",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "bucket/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "bucket/newfolder/afile",
+			size: 4,
+		},
+		{
+			name: "bucket/newfolder/anotherone",
+			size: 1,
+		},
+		{
+			name: "bucket/newfolder/anemptyone",
+			size: 0,
+		},
+		{
+			name: "bucket/dir1/fileindir1",
+			size: 20000,
+		},
+		{
+			name: "bucket/dir1/dirc/fileindirc",
+			size: 20000,
+		},
+		{
+			name: "bucket/rootfile3",
+			size: 1000,
+		},
+	}
+	createUsageTestFiles(t, base, files)
+	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   1363315,
+			objs:   14,
+			oSizes: sizeHistogram{0: 6, 1: 8},
+		},
+		{
+			path:   "bucket/",
+			size:   21000,
+			objs:   3,
+			oSizes: sizeHistogram{0: 1, 1: 2},
+		},
+		{
+			path:   "bucket/newfolder",
+			size:   5,
+			objs:   3,
+			oSizes: sizeHistogram{0: 3},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "bucket/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "bucket/dir1/dira/dirasub/fileindira2",
+			size: 200,
+		},
+	}
+
+	createUsageTestFiles(t, base, files)
+	err = os.RemoveAll(filepath.Join(base, "bucket/dir1/dira/dirasub/dcfile"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Changed dir must be picked up in this many cycles.
+	for i := 0; i < dataUsageUpdateDirCycles; i++ {
+		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   363515,
+			objs:   14,
+			oSizes: sizeHistogram{0: 7, 1: 7},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   300210,
+			objs:   4,
+			oSizes: sizeHistogram{0: 2, 1: 2},
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+}
+
+func createUsageTestFiles(t *testing.T, base string, files []usageTestFile) {
+	for _, f := range files {
+		err := os.MkdirAll(filepath.Dir(filepath.Join(base, f.name)), os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ioutil.WriteFile(filepath.Join(base, f.name), make([]byte, f.size), os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestDataUsageCacheSerialize(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUsageCacheSerialize")
+	if err != nil {
+		t.Skip(err)
+	}
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "rootfile", size: 10000},
+		{name: "rootfile2", size: 10000},
+		{name: "dir1/d1file", size: 2000},
+		{name: "dir2/d2file", size: 300},
+		{name: "dir1/dira/dafile", size: 100000},
+		{name: "dir1/dira/dbfile", size: 200000},
+		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+	want, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := want.serialize()
+	var got dataUsageCache
+	err = got.deserialize(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got.Info.LastUpdate.IsZero() {
+		t.Error("lastupdate not set")
+	}
+
+	if !want.Info.LastUpdate.Equal(got.Info.LastUpdate) {
+		t.Fatalf("deserialize mismatch\nwant: %+v\ngot:  %+v", want, got)
+	}
+}