Avoid double for loops in notification init (#8691)

The output is directly borrowed from `mc` command output

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.github

.github/logo.svg

@@ -0,0 +1 @@
+<svg data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 162.612 24.465"><path d="M52.751.414h9.108v23.63h-9.108zM41.711.74l-18.488 9.92a.919.919 0 0 1-.856 0L3.879.74A2.808 2.808 0 0 0 2.558.414h-.023A2.4 2.4 0 0 0 0 2.641v21.376h9.1V13.842a.918.918 0 0 1 1.385-.682l10.361 5.568a3.634 3.634 0 0 0 3.336.028l10.933-5.634a.917.917 0 0 1 1.371.69v10.205h9.1V2.641A2.4 2.4 0 0 0 43.055.414h-.023a2.808 2.808 0 0 0-1.321.326zm65.564-.326h-9.237v10.755a.913.913 0 0 1-1.338.706L72.762.675a2.824 2.824 0 0 0-1.191-.261h-.016a2.4 2.4 0 0 0-2.535 2.227v21.377h9.163V13.275a.914.914 0 0 1 1.337-.707l24.032 11.2a2.813 2.813 0 0 0 1.188.26 2.4 2.4 0 0 0 2.535-2.227zm7.161 23.63V.414h4.191v23.63zm28.856.421c-11.274 0-19.272-4.7-19.272-12.232C124.02 4.741 132.066 0 143.292 0s19.32 4.7 19.32 12.233-7.902 12.232-19.32 12.232zm0-21.333c-8.383 0-14.84 3.217-14.84 9.1 0 5.926 6.457 9.1 14.84 9.1s14.887-3.174 14.887-9.1c0-5.883-6.504-9.1-14.887-9.1z" fill="#c72c48"/></svg>

.travis.yml

@@ -26,18 +26,17 @@ matrix:
         - ARCH=x86_64
         - CGO_ENABLED=0
         - GO111MODULE=on
-        - GOPROXY=https://proxy.golang.org
-      go: 1.12.x
+        - SIMPLE_CI=1
+      go: 1.13.x
       script:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
         - diff -au <(gofmt -s -d pkg) <(printf "")
-        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 15m "$d"; done
+        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done
         - make verifiers
         - make crosscompile
         - make verify
-        - make coverage
-        - cd browser && yarn && yarn test && cd ..
+        - cd browser && npm install && npm run test && cd ..
         - bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'
 
     - os: windows
@@ -45,11 +44,11 @@ matrix:
         - ARCH=x86_64
         - CGO_ENABLED=0
         - GO111MODULE=on
-        - GOPROXY=https://proxy.golang.org
-      go: 1.12.x
+        - SIMPLE_CI=1
+      go: 1.13.x
       script:
         - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
-        - bash buildscripts/go-coverage.sh
+        - for d in $(go list ./... | grep -v browser); do go test -v --timeout 20m "$d"; done
 
 before_script:
   # Add an IPv6 config - see the corresponding Travis issue
@@ -57,7 +56,4 @@ before_script:
   - if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi
 
 before_install:
- - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install 11.10.1 ; fi
-
-after_success:
-- bash <(curl -s https://codecov.io/bash)
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi

Dockerfile

@@ -1,10 +1,11 @@
-FROM golang:1.12-alpine
+FROM golang:1.13-alpine
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 ENV GOPATH /go
 ENV CGO_ENABLED 0
 ENV GO111MODULE on
+ENV GOPROXY https://proxy.golang.org
 
 RUN  \
      apk add --no-cache git && \
@@ -16,7 +17,7 @@ FROM alpine:3.9
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
 
 EXPOSE 9000
 

Dockerfile.dev

@@ -8,7 +8,7 @@ COPY minio /usr/bin/
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \

Dockerfile.release

@@ -1,4 +1,4 @@
-FROM golang:1.12-alpine
+FROM golang:1.13-alpine
 
 ENV GOPATH /go
 ENV CGO_ENABLED 0
@@ -17,7 +17,7 @@ COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \

Dockerfile.simpleci

@@ -1,14 +1,14 @@
 #-------------------------------------------------------------
 # Stage 1: Build and Unit tests
 #-------------------------------------------------------------
-FROM golang:1.12
+FROM golang:1.13
 
 COPY . /go/src/github.com/minio/minio
 WORKDIR /go/src/github.com/minio/minio
 
 RUN apt-get update && apt-get install -y jq
 ENV GO111MODULE=on
-ENV GOPROXY=https://proxy.golang.org
+ENV SIMPLE_CI 1
 
 RUN git config --global http.cookiefile /gitcookie/.gitcookie
 
@@ -44,6 +44,8 @@ RUN make verify
 #-------------------------------------------------------------
 FROM node:10.15-stretch-slim
 
+ENV SIMPLE_CI 1
+
 COPY browser /minio/browser
 WORKDIR /minio/browser
 
@@ -64,6 +66,7 @@ ENV LANG C.UTF-8
 ENV GOROOT /usr/local/go
 ENV GOPATH /usr/local
 ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+ENV SIMPLE_CI 1
 ENV MINT_ROOT_DIR /mint
 
 RUN apt-get --yes update && apt-get --yes upgrade && \

MAINTAINERS.md

@@ -1,37 +0,0 @@
-# For maintainers only
-
-### Setup your minio GitHub Repository
-
-Fork [minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository.
-```bash
-$ mkdir -p $GOPATH/src/github.com/minio
-$ cd $GOPATH/src/github.com/minio
-$ git clone https://github.com/$USER_ID/minio
-$ 
-```
-
-``minio`` uses [govendor](https://github.com/kardianos/govendor) for its dependency management.
-
-### To manage dependencies
-
-#### Add new dependencies
-
-  - Run `go get foo/bar`
-  - Edit your code to import foo/bar
-  - Run `govendor add foo/bar` from top-level directory
-
-#### Remove dependencies 
-
-  - Run `govendor remove foo/bar`
-
-#### Update dependencies
-
-  - Run `govendor remove +vendor`
-  - Run to update the dependent package `go get -u foo/bar`
-  - Run `govendor add +external`
-
-### Making new releases 
-
-`minio` doesn't follow semantic versioning style, `minio` instead uses the release date and time as the release versions.
-
-`make release` will generate new binary into `release` directory.

Makefile

@@ -2,13 +2,12 @@ PWD := $(shell pwd)
 GOPATH := $(shell go env GOPATH)
 LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 
+GOARCH := $(shell go env GOARCH)
 GOOS := $(shell go env GOOS)
-GOOSALT ?= 'linux'
-ifeq ($(GOOS),'darwin')
-  GOOSALT = 'mac'
-endif
 
-TAG ?= $(USER)
+VERSION ?= $(shell git describe --tags)
+TAG ?= "minio/minio:$(VERSION)"
+
 BUILD_LDFLAGS := '$(LDFLAGS)'
 
 all: build
@@ -19,9 +18,13 @@ checks:
 
 getdeps:
 	@mkdir -p ${GOPATH}/bin
-	@which golint 1>/dev/null || (echo "Installing golint" && go get -u golang.org/x/lint/golint)
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet -O ${GOPATH}/bin/staticcheck https://github.com/dominikh/go-tools/releases/download/2019.1/staticcheck_${GOOS}_amd64 && chmod +x ${GOPATH}/bin/staticcheck)
-	@which misspell 1>/dev/null || (echo "Installing misspell" && wget --quiet https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_${GOOSALT}_64bit.tar.gz && tar xf misspell_0.3.4_${GOOSALT}_64bit.tar.gz && mv misspell ${GOPATH}/bin/misspell && chmod +x ${GOPATH}/bin/misspell && rm -f misspell_0.3.4_${GOOSALT}_64bit.tar.gz)
+	@which golint 1>/dev/null || (echo "Installing golint" && GO111MODULE=off go get -u golang.org/x/lint/golint)
+ifeq ($(GOARCH),s390x)
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && GO111MODULE=off go get honnef.co/go/tools/cmd/staticcheck)
+else
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2019.2.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
+endif
+	@which misspell 1>/dev/null || (echo "Installing misspell" && GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
@@ -29,39 +32,42 @@ crosscompile:
 verifiers: getdeps vet fmt lint staticcheck spelling
 
 vet:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on go vet github.com/minio/minio/...
+	@echo "Running $@ check"
+	@GO111MODULE=on go vet github.com/minio/minio/...
 
 fmt:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on gofmt -d cmd/
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on gofmt -d pkg/
+	@echo "Running $@ check"
+	@GO111MODULE=on gofmt -d cmd/
+	@GO111MODULE=on gofmt -d pkg/
 
 lint:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
+	@echo "Running $@ check"
+	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
+	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
 
 staticcheck:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
+	@echo "Running $@ check"
+	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
+	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
 
 spelling:
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find cmd/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find pkg/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find docs/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
+	@echo "Running $@ check"
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find cmd/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find pkg/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find docs/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build
 	@echo "Running unit tests"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
-verify: build
-	@echo "Verifying build"
+# Verify minio binary
+verify:
+	@echo "Verifying build with race"
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 coverage: build
@@ -71,7 +77,7 @@ coverage: build
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

README.md

@@ -1,14 +1,9 @@
 # MinIO Quickstart Guide
 [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
 
-MinIO is an object storage server released under Apache License v2.0. It is compatible[1] with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
+[![MinIO](https://raw.githubusercontent.com/minio/minio/master/.github/logo.svg?sanitize=true)](https://min.io)
 
-MinIO server is light enough to be bundled with the application stack, similar to NodeJS, Redis and MySQL.
-
-[1]: MinIO in its default mode is faster and does not calculate MD5Sum unless passed by client. This may lead to incompatibility with few S3 clients like s3ql that heavily depend on MD5Sum. For full compatibility with Amazon S3 API, start MinIO with `--compat` option.
-```sh
-minio --compat server /data
-```
+MinIO is High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Using MinIO build high performance infrastructure for machine learning, analytics and application data workloads.
 
 ## Docker Container
 ### Stable
@@ -22,7 +17,8 @@ docker run -p 9000:9000 minio/minio server /data
 docker pull minio/minio:edge
 docker run -p 9000:9000 minio/minio:edge server /data
 ```
-Note: Docker will not display the autogenerated keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use autogenerated keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
+
+> NOTE: Docker will not display the default keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use default keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
 ### Homebrew
@@ -39,8 +35,8 @@ brew install minio/stable/minio
 ```
 
 ### Binary Download
-| Platform    | Architecture | URL                                                         |
-| ----------  | --------     | ------                                                      |
+| Platform    | Architecture | URL                                                       |
+| ----------  | --------     | ------                                                    |
 | Apple macOS | 64-bit Intel | https://dl.min.io/server/minio/release/darwin-amd64/minio |
 ```sh
 chmod 755 minio
@@ -49,8 +45,8 @@ chmod 755 minio
 
 ## GNU/Linux
 ### Binary Download
-| Platform   | Architecture | URL                                                        |
-| ---------- | --------     | ------                                                     |
+| Platform   | Architecture | URL                                                      |
+| ---------- | --------     | ------                                                   |
 | GNU/Linux  | 64-bit Intel | https://dl.min.io/server/minio/release/linux-amd64/minio |
 ```sh
 wget https://dl.min.io/server/minio/release/linux-amd64/minio
@@ -58,8 +54,8 @@ chmod +x minio
 ./minio server /data
 ```
 
-| Platform   | Architecture | URL                                                          |
-| ---------- | --------     | ------                                                       |
+| Platform   | Architecture | URL                                                        |
+| ---------- | --------     | ------                                                     |
 | GNU/Linux  | ppc64le      | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
 ```sh
 wget https://dl.min.io/server/minio/release/linux-ppc64le/minio
@@ -69,8 +65,8 @@ chmod +x minio
 
 ## Microsoft Windows
 ### Binary Download
-| Platform          | Architecture | URL                                                              |
-| ----------        | --------     | ------                                                           |
+| Platform          | Architecture | URL                                                            |
+| ----------        | --------     | ------                                                         |
 | Microsoft Windows | 64-bit       | https://dl.min.io/server/minio/release/windows-amd64/minio.exe |
 ```sh
 minio.exe server D:\Photos
@@ -88,10 +84,10 @@ service minio start
 ```
 
 ## Install from Source
-Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.12](https://golang.org/dl/#stable)
+Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.13](https://golang.org/dl/#stable)
 
 ```sh
-GOPROXY=https://proxy.golang.org GO111MODULE=on go get github.com/minio/minio
+GO111MODULE=on go get github.com/minio/minio
 ```
 
 ## Allow port access for Firewalls
@@ -162,6 +158,22 @@ When deployed on a single drive, MinIO server lets clients access any pre-existi
 
 The above statement is also valid for all gateway backends.
 
+## Upgrading MinIO
+MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster and restart them, as shown in the following command from the MinIO client (mc):
+
+```
+mc admin update <minio alias, e.g., myminio>
+```
+
+**Important things to remember during upgrades**:
+
+- `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
+- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` until all clusters have been updated.
+- If you are updating the server it is always recommended (unless explicitly mentioned in MinIO server release notes), to update `mc` once all the servers have been upgraded using `mc update`.
+- `mc admin update` is disabled in docker/container environments, container environments provide their own mechanisms for updating running containers.
+- If you are using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
+- If you are using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
+
 ## Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
 - [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
@@ -173,5 +185,11 @@ The above statement is also valid for all gateway backends.
 ## Contribute to MinIO Project
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
+## Caveats
+MinIO in its default mode doesn't use MD5Sum checkums of incoming streams unless requested by the client in `Content-Md5` header for validation. This may lead to incompatibility with rare S3 clients like `s3ql` which unfortunately do not set `Content-Md5` but depend on hex MD5Sum for the stream to be calculated by the server. MinIO considers this as a bug in `s3ql` and should be fixed on the client side because MD5Sum is a poor way to checksum and validate the authenticity of the objects. Although MinIO provides a workaround until client applications are fixed use `--compat` option instead to start the server.
+```sh
+./minio --compat server /data
+```
+
 ## License
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fminio%2Fminio.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fminio%2Fminio?ref=badge_large)

browser/app/js/App.js

@@ -18,11 +18,13 @@ import React from "react"
 import { Route, Switch, Redirect } from "react-router-dom"
 import Browser from "./browser/Browser"
 import Login from "./browser/Login"
+import OpenIDLogin from "./browser/OpenIDLogin"
 import web from "./web"
 
 export const App = () => {
   return (
     <Switch>
+      <Route path={"/login/openid"} component={OpenIDLogin} />
       <Route path={"/login"} component={Login} />
       <Route path={"/:bucket?/*"} component={Browser} />
     </Switch>

browser/app/js/browser/AboutModal.js

@@ -19,7 +19,7 @@ import { Modal } from "react-bootstrap"
 import logo from "../../img/logo.svg"
 
 export const AboutModal = ({ serverInfo, hideAbout }) => {
-  const { version, memory, platform, runtime } = serverInfo
+  const { version, platform, runtime } = serverInfo
   return (
     <Modal
       className="modal-about modal-dark"
@@ -42,10 +42,6 @@ export const AboutModal = ({ serverInfo, hideAbout }) => {
               <div>Version</div>
               <small>{version}</small>
             </li>
-            <li>
-              <div>Memory</div>
-              <small>{memory}</small>
-            </li>
             <li>
               <div>Platform</div>
               <small>{platform}</small>

browser/app/js/browser/ChangePasswordModal.js

@@ -80,13 +80,6 @@ export class ChangePasswordModal extends React.Component {
 
   generateAuth(e) {
     const { serverInfo } = this.props
-    // Generate random access key only for root user
-    if (!serverInfo.userInfo.isIAMUser) {
-      this.setState({
-        newAccessKey: getRandomAccessKey()
-      })
-    }
-
     this.setState({
       newSecretKey: getRandomSecretKey(),
       newSecretKeyVisible: true
@@ -100,10 +93,16 @@ export class ChangePasswordModal extends React.Component {
       return false
     }
 
-    // When credentials are set on ENV, password change not allowed for owner
-    if (serverInfo.info.isEnvCreds && !serverInfo.userInfo.isIAMUser) {
+    // Password change is not allowed for temporary users(STS)
+    if(serverInfo.userInfo.isTempUser) {
       return false
     }
+
+    // Password change is only allowed for regular users
+    if (!serverInfo.userInfo.isIAMUser) {
+      return false
+    }
+
     return true
   }
 
@@ -186,24 +185,6 @@ export class ChangePasswordModal extends React.Component {
           </div>
 
           <div className="has-toggle-password m-t-30">
-            {!serverInfo.userInfo.isIAMUser && (
-              <InputGroup
-                value={this.state.newAccessKey}
-                id="newAccessKey"
-                label={"New Access Key"}
-                name="newAccesskey"
-                type="text"
-                spellCheck="false"
-                required="required"
-                autoComplete="false"
-                align="ig-left"
-                onChange={e => {
-                  this.setState({ newAccessKey: e.target.value })
-                }}
-                readonly={serverInfo.userInfo.isIAMUser}
-              />
-            )}
-
             <i
               onClick={() => {
                 this.setState({

browser/app/js/browser/Login.js

@@ -16,20 +16,22 @@
 
 import React from "react"
 import { connect } from "react-redux"
-import classNames from "classnames"
 import logo from "../../img/logo.svg"
 import Alert from "../alert/Alert"
 import * as actionsAlert from "../alert/actions"
 import InputGroup from "./InputGroup"
 import web from "../web"
-import { Redirect } from "react-router-dom"
+import { Redirect, Link } from "react-router-dom"
+import OpenIDLoginButton from './OpenIDLoginButton'
 
 export class Login extends React.Component {
   constructor(props) {
     super(props)
     this.state = {
       accessKey: "",
-      secretKey: ""
+      secretKey: "",
+      discoveryDoc: {},
+      clientId: ""
     }
   }
 
@@ -83,6 +85,15 @@ export class Login extends React.Component {
     document.body.classList.add("is-guest")
   }
 
+  componentDidMount() {
+    web.GetDiscoveryDoc().then(({ DiscoveryDoc, clientId }) => {
+      this.setState({
+        clientId,
+        discoveryDoc: DiscoveryDoc
+      })
+    })
+  }
+
   componentWillUnmount() {
     document.body.classList.remove("is-guest")
   }
@@ -95,6 +106,8 @@ export class Login extends React.Component {
     let alertBox = <Alert {...alert} onDismiss={clearAlert} />
     // Make sure you don't show a fading out alert box on the initial web-page load.
     if (!alert.message) alertBox = ""
+
+    const showOpenID = Boolean(this.state.discoveryDoc && this.state.discoveryDoc.authorization_endpoint)
     return (
       <div className="login">
         {alertBox}
@@ -122,12 +135,31 @@ export class Login extends React.Component {
               type="password"
               spellCheck="false"
               required="required"
-              autoComplete="new-password"
             />
             <button className="lw-btn" type="submit">
               <i className="fas fa-sign-in-alt" />
             </button>
           </form>
+          {showOpenID && (
+            <div className="openid-login">
+              <div className="or">or</div>
+              {
+                this.state.clientId ? (
+                  <OpenIDLoginButton
+                    className="btn openid-btn"
+                    clientId={this.state.clientId}
+                    authorizationEndpoint={this.state.discoveryDoc.authorization_endpoint}
+                  >
+                    Log in with OpenID
+                  </OpenIDLoginButton>
+                ) : (
+                  <Link to={"/login/openid"} className="btn openid-btn">
+                    Log in with OpenID
+                  </Link>
+                )
+              }
+            </div>
+          )}
         </div>
         <div className="l-footer">
           <a className="lf-logo" href="">

browser/app/js/browser/OpenIDLogin.js

@@ -0,0 +1,168 @@
+/*
+ * MinIO Cloud Storage (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { connect } from "react-redux"
+import logo from "../../img/logo.svg"
+import Alert from "../alert/Alert"
+import * as actionsAlert from "../alert/actions"
+import InputGroup from "./InputGroup"
+import web from "../web"
+import { Redirect } from "react-router-dom"
+import qs from "query-string"
+import { getRandomString } from "../utils"
+import storage from "local-storage-fallback"
+import jwtDecode from "jwt-decode"
+import { buildOpenIDAuthURL, OPEN_ID_NONCE_KEY } from './utils'
+
+export class OpenIDLogin extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      clientID: "",
+      discoveryDoc: {}
+    }
+    this.clientIDChange = this.clientIDChange.bind(this)
+    this.handleSubmit = this.handleSubmit.bind(this)
+  }
+
+  clientIDChange(e) {
+    this.setState({
+      clientID: e.target.value
+    })
+  }
+
+  handleSubmit(event) {
+    event.preventDefault()
+    const { showAlert } = this.props
+    let message = ""
+    if (this.state.clientID === "") {
+      message = "Client ID cannot be empty"
+    }
+    if (message) {
+      showAlert("danger", message)
+      return
+    }
+
+    if (this.state.discoveryDoc && this.state.discoveryDoc.authorization_endpoint) {
+      const redirectURI = window.location.href.split("#")[0]
+
+      // Store nonce in localstorage to check again after the redirect
+      const nonce = getRandomString(16)
+      storage.setItem(OPEN_ID_NONCE_KEY, nonce)
+
+      const authURL = buildOpenIDAuthURL(
+        this.state.discoveryDoc.authorization_endpoint,
+        redirectURI,
+        this.state.clientID,
+        nonce
+      )
+      window.location = authURL
+    }
+  }
+
+  componentWillMount() {
+    const { clearAlert } = this.props
+    // Clear out any stale message in the alert of previous page
+    clearAlert()
+    document.body.classList.add("is-guest")
+
+    web.GetDiscoveryDoc().then(({ DiscoveryDoc }) => {
+      this.setState({
+        discoveryDoc: DiscoveryDoc
+      })
+    })
+  }
+
+  componentDidMount() {
+    const values = qs.parse(this.props.location.hash)
+    if (values.error) {
+      this.props.showAlert("danger", values.error_description)
+      return
+    }
+
+    if (values.id_token) {
+      // Check nonce on the token to prevent replay attacks
+      const tokenJSON = jwtDecode(values.id_token)
+      if (storage.getItem(OPEN_ID_NONCE_KEY) !== tokenJSON.nonce) {
+        this.props.showAlert("danger", "Invalid auth token")
+        return
+      }
+
+      web.LoginSTS({ token: values.id_token }).then(() => {
+        storage.removeItem(OPEN_ID_NONCE_KEY)
+        this.forceUpdate()
+        return
+      })
+    }
+  }
+
+  componentWillUnmount() {
+    document.body.classList.remove("is-guest")
+  }
+
+  render() {
+    const { clearAlert, alert } = this.props
+    if (web.LoggedIn()) {
+      return <Redirect to={"/"} />
+    }
+    let alertBox = <Alert {...alert} onDismiss={clearAlert} />
+    // Make sure you don't show a fading out alert box on the initial web-page load.
+    if (!alert.message) alertBox = ""
+    return (
+      <div className="login">
+        {alertBox}
+        <div className="l-wrap">
+          <form onSubmit={this.handleSubmit}>
+            <InputGroup
+              value={this.state.clientID}
+              onChange={this.clientIDChange}
+              className="ig-dark"
+              label="Client ID"
+              id="clientID"
+              name="clientID"
+              type="text"
+              spellCheck="false"
+              required="required"
+            />
+            <button className="lw-btn" type="submit">
+              <i className="fas fa-sign-in-alt" />
+            </button>
+          </form>
+        </div>
+        <div className="l-footer">
+          <a className="lf-logo" href="">
+            <img src={logo} alt="" />
+          </a>
+          <div className="lf-server">{window.location.host}</div>
+        </div>
+      </div>
+    )
+  }
+}
+
+const mapDispatchToProps = dispatch => {
+  return {
+    showAlert: (type, message) =>
+      dispatch(actionsAlert.set({ type: type, message: message })),
+    clearAlert: () => dispatch(actionsAlert.clear())
+  }
+}
+
+export default connect(
+  state => state,
+  mapDispatchToProps
+)(OpenIDLogin)

browser/app/js/browser/OpenIDLoginButton.js

@@ -0,0 +1,57 @@
+/*
+ * MinIO Cloud Storage (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { getRandomString } from "../utils"
+import storage from "local-storage-fallback"
+import { buildOpenIDAuthURL, OPEN_ID_NONCE_KEY } from './utils'
+
+export class OpenIDLoginButton extends React.Component {
+  constructor(props) {
+    super(props)
+    this.handleClick = this.handleClick.bind(this)
+  }
+
+  handleClick(event) {
+    event.stopPropagation()
+    const { authorizationEndpoint, clientId } = this.props
+
+    let redirectURI = window.location.href.split("#")[0]
+    if (redirectURI.endsWith('/')) {
+      redirectURI += 'openid'
+    } else {
+      redirectURI += '/openid'
+    }
+
+    // Store nonce in localstorage to check again after the redirect
+    const nonce = getRandomString(16)
+    storage.setItem(OPEN_ID_NONCE_KEY, nonce)
+
+    const authURL = buildOpenIDAuthURL(authorizationEndpoint, redirectURI, clientId, nonce)
+    window.location = authURL
+  }
+
+  render() {
+    const { children, className } = this.props
+    return (
+      <div onClick={this.handleClick} className={className}>
+        {children}
+      </div>
+    )
+  }
+}
+
+export default OpenIDLoginButton

browser/app/js/browser/StorageInfo.js

@@ -26,6 +26,13 @@ export class StorageInfo extends React.Component {
   }
   render() {
     const { used } = this.props.storageInfo
+
+    if (!used) {
+      return <noscript />
+    }
+
+    const totalUsed = used.reduce((v1, v2) => v1 + v2, 0)
+
     return (
       <div className="feh-used">
         <div className="fehu-chart">
@@ -34,7 +41,7 @@ export class StorageInfo extends React.Component {
         <ul>
           <li>
             <span>Used: </span>
-            {humanize.filesize(used)}
+            {humanize.filesize(totalUsed)}
           </li>
         </ul>
       </div>
@@ -54,4 +61,7 @@ const mapDispatchToProps = dispatch => {
   }
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(StorageInfo)
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(StorageInfo)

browser/app/js/browser/__tests__/AboutModal.test.js

@@ -21,7 +21,6 @@ import { AboutModal } from "../AboutModal"
 describe("AboutModal", () => {
   const serverInfo = {
     version: "test",
-    memory: "test",
     platform: "test",
     runtime: "test"
   }

browser/app/js/browser/__tests__/BrowserDropdown.test.js

@@ -21,7 +21,6 @@ import { BrowserDropdown } from "../BrowserDropdown"
 describe("BrowserDropdown", () => {
   const serverInfo = {
     version: "test",
-    memory: "test",
     platform: "test",
     runtime: "test"
   }

browser/app/js/browser/__tests__/ChangePasswordModal.test.js

@@ -54,11 +54,10 @@ jest.mock("../../utils", () => ({
 describe("ChangePasswordModal", () => {
   const serverInfo = {
     version: "test",
-    memory: "test",
     platform: "test",
     runtime: "test",
-    info: { isEnvCreds: false },
-    userInfo: { isIAMUser: false }
+    info: {},
+    userInfo: { isIAMUser: true }
   }
 
   it("should render without crashing", () => {
@@ -76,10 +75,9 @@ describe("ChangePasswordModal", () => {
     ).toBe("Credentials of this user cannot be updated through MinIO Browser.")
   })
 
-  it("should not allow changing password when isEnvCreds is true and not IAM user", () => {
+  it("should not allow changing password when not IAM user", () => {
     const newServerInfo = {
       ...serverInfo,
-      info: { isEnvCreds: true },
       userInfo: { isIAMUser: false }
     }
     const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
@@ -91,21 +89,22 @@ describe("ChangePasswordModal", () => {
     ).toBe("Credentials of this user cannot be updated through MinIO Browser.")
   })
 
-  it("should generate accessKey and secretKey when Generate buttons is clicked", () => {
-    const wrapper = shallow(<ChangePasswordModal serverInfo={serverInfo} />)
-    wrapper.find("#generate-keys").simulate("click")
-    setImmediate(() => {
-      expect(wrapper.state("newAccessKey")).toBe("raccesskey")
-      expect(wrapper.state("newSecretKey")).toBe("rsecretkey")
-    })
+  it("should not allow changing password for STS user", () => {
+    const newServerInfo = {
+      ...serverInfo,
+      userInfo: { isTempUser: true }
+    }
+    const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
+    expect(
+      wrapper
+        .find("ModalBody")
+        .childAt(0)
+        .text()
+    ).toBe("Credentials of this user cannot be updated through MinIO Browser.")
   })
 
   it("should not generate accessKey for IAM User", () => {
-    const newServerInfo = {
-      ...serverInfo,
-      userInfo: { isIAMUser: true }
-    }
-    const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
+    const wrapper = shallow(<ChangePasswordModal serverInfo={serverInfo} />)
     wrapper.find("#generate-keys").simulate("click")
     setImmediate(() => {
       expect(wrapper.state("newAccessKey")).toBe("minio")
@@ -114,59 +113,24 @@ describe("ChangePasswordModal", () => {
   })
 
   it("should not show new accessKey field for IAM User", () => {
-    const newServerInfo = {
-      ...serverInfo,
-      userInfo: { isIAMUser: true }
-    }
-    const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
+    const wrapper = shallow(<ChangePasswordModal serverInfo={serverInfo} />)
     expect(wrapper.find("#newAccesskey").exists()).toBeFalsy()
   })
 
-  it("should disble Update button for invalid accessKey or secretKey", () => {
+  it("should disable Update button for secretKey", () => {
     const showAlert = jest.fn()
     const wrapper = shallow(
       <ChangePasswordModal serverInfo={serverInfo} showAlert={showAlert} />
     )
-    wrapper
-      .find("#currentAccessKey")
-      .simulate("change", { target: { value: "minio" } })
     wrapper
       .find("#currentSecretKey")
       .simulate("change", { target: { value: "minio123" } })
-    wrapper.find("#newAccessKey").simulate("change", { target: { value: "t" } })
     wrapper
       .find("#newSecretKey")
       .simulate("change", { target: { value: "t1" } })
     expect(wrapper.find("#update-keys").prop("disabled")).toBeTruthy()
   })
 
-  it("should update accessKey and secretKey when Update button is clicked", () => {
-    const showAlert = jest.fn()
-    const wrapper = shallow(
-      <ChangePasswordModal serverInfo={serverInfo} showAlert={showAlert} />
-    )
-    wrapper
-      .find("#currentAccessKey")
-      .simulate("change", { target: { value: "minio" } })
-    wrapper
-      .find("#currentSecretKey")
-      .simulate("change", { target: { value: "minio123" } })
-    wrapper
-      .find("#newAccessKey")
-      .simulate("change", { target: { value: "test" } })
-    wrapper
-      .find("#newSecretKey")
-      .simulate("change", { target: { value: "test1234" } })
-    expect(wrapper.find("#update-keys").prop("disabled")).toBeFalsy()
-    wrapper.find("#update-keys").simulate("click")
-    setImmediate(() => {
-      expect(showAlert).toHaveBeenCalledWith({
-        type: "success",
-        message: "Credentials updated successfully."
-      })
-    })
-  })
-
   it("should call hideChangePassword when Cancel button is clicked", () => {
     const hideChangePassword = jest.fn()
     const wrapper = shallow(

browser/app/js/browser/__tests__/Login.test.js

@@ -19,21 +19,24 @@ import { shallow, mount } from "enzyme"
 import { Login } from "../Login"
 import web from "../../web"
 
-jest.mock('../../web', () => ({
+jest.mock("../../web", () => ({
   Login: jest.fn(() => {
     return Promise.resolve({ token: "test", uiVersion: "2018-02-01T01:17:47Z" })
   }),
-  LoggedIn: jest.fn()
+  LoggedIn: jest.fn(),
+  GetDiscoveryDoc: jest.fn(() => {
+    return Promise.resolve({ DiscoveryDoc: {"authorization_endpoint": "test"} })
+  })
 }))
 
 describe("Login", () => {
   const dispatchMock = jest.fn()
   const showAlertMock = jest.fn()
   const clearAlertMock = jest.fn()
-  
+
   it("should render without crashing", () => {
-    shallow(<Login 
-      dispatch={dispatchMock} 
+    shallow(<Login
+      dispatch={dispatchMock}
       alert={{ show: false, type: "danger"}}
       showAlert={showAlertMock}
       clearAlert={clearAlertMock}
@@ -42,8 +45,8 @@ describe("Login", () => {
 
   it("should initially have the is-guest class", () => {
     const wrapper = shallow(
-      <Login 
-        dispatch={dispatchMock} 
+      <Login
+        dispatch={dispatchMock}
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
@@ -55,8 +58,8 @@ describe("Login", () => {
 
   it("should throw an alert if the keys are empty in login form", () => {
     const wrapper = mount(
-      <Login 
-        dispatch={dispatchMock} 
+      <Login
+        dispatch={dispatchMock}
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
@@ -85,8 +88,8 @@ describe("Login", () => {
 
   it("should call web.Login with correct arguments if both keys are entered", () => {
     const wrapper = mount(
-      <Login 
-        dispatch={dispatchMock} 
+      <Login
+        dispatch={dispatchMock}
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
@@ -98,7 +101,7 @@ describe("Login", () => {
     })
     wrapper.find("form").simulate("submit")
     expect(web.Login).toHaveBeenCalledWith({
-      "username": "accessKey", 
+      "username": "accessKey",
       "password": "secretKey"
     })
   })

browser/app/js/browser/__tests__/StorageInfo.test.js

@@ -21,10 +21,7 @@ import { StorageInfo } from "../StorageInfo"
 describe("StorageInfo", () => {
   it("should render without crashing", () => {
     shallow(
-      <StorageInfo
-        storageInfo={{ used: 60 }}
-        fetchStorageInfo={jest.fn()}
-      />
+      <StorageInfo storageInfo={{ used: [60] }} fetchStorageInfo={jest.fn()} />
     )
   })
 
@@ -32,10 +29,21 @@ describe("StorageInfo", () => {
     const fetchStorageInfo = jest.fn()
     shallow(
       <StorageInfo
-        storageInfo={{ used: 60 }}
+        storageInfo={{ used: [60] }}
         fetchStorageInfo={fetchStorageInfo}
       />
     )
     expect(fetchStorageInfo).toHaveBeenCalled()
   })
+
+  it("should not render anything if used is null", () => {
+    const fetchStorageInfo = jest.fn()
+    const wrapper = shallow(
+      <StorageInfo
+        storageInfo={{ used: null }}
+        fetchStorageInfo={fetchStorageInfo}
+      />
+    )
+    expect(wrapper.text()).toBe("")
+  })
 })

browser/app/js/browser/__tests__/actions.test.js

@@ -20,12 +20,11 @@ import * as actionsCommon from "../actions"
 
 jest.mock("../../web", () => ({
   StorageInfo: jest.fn(() => {
-    return Promise.resolve({ storageInfo: { Used: 60 } })
+    return Promise.resolve({ storageInfo: { Used: [60] } })
   }),
   ServerInfo: jest.fn(() => {
     return Promise.resolve({
       MinioVersion: "test",
-      MinioMemory: "test",
       MinioPlatform: "test",
       MinioRuntime: "test",
       MinioGlobalInfo: "test"
@@ -40,7 +39,7 @@ describe("Common actions", () => {
   it("creates common/SET_STORAGE_INFO after fetching the storage details ", () => {
     const store = mockStore()
     const expectedActions = [
-      { type: "common/SET_STORAGE_INFO", storageInfo: { used: 60 } }
+      { type: "common/SET_STORAGE_INFO", storageInfo: { used: [60] } }
     ]
     return store.dispatch(actionsCommon.fetchStorageInfo()).then(() => {
       const actions = store.getActions()
@@ -55,7 +54,6 @@ describe("Common actions", () => {
         type: "common/SET_SERVER_INFO",
         serverInfo: {
           version: "test",
-          memory: "test",
           platform: "test",
           runtime: "test",
           info: "test"

browser/app/js/browser/__tests__/reducer.test.js

@@ -22,8 +22,9 @@ describe("common reducer", () => {
     expect(reducer(undefined, {})).toEqual({
       sidebarOpen: false,
       storageInfo: {
-        total: 0,
-        free: 0
+        total: [0],
+        free: [0],
+        used: [0]
       },
       serverInfo: {}
     })
@@ -61,11 +62,11 @@ describe("common reducer", () => {
         {},
         {
           type: actionsCommon.SET_STORAGE_INFO,
-          storageInfo: { total: 100, free: 40 }
+          storageInfo: { total: [100], free: [40] }
         }
       )
     ).toEqual({
-      storageInfo: { total: 100, free: 40 }
+      storageInfo: { total: [100], free: [40] }
     })
   })
 
@@ -75,7 +76,6 @@ describe("common reducer", () => {
         type: actionsCommon.SET_SERVER_INFO,
         serverInfo: {
           version: "test",
-          memory: "test",
           platform: "test",
           runtime: "test",
           info: "test"
@@ -83,7 +83,6 @@ describe("common reducer", () => {
       }).serverInfo
     ).toEqual({
       version: "test",
-      memory: "test",
       platform: "test",
       runtime: "test",
       info: "test"

browser/app/js/browser/actions.js

@@ -51,7 +51,6 @@ export const fetchServerInfo = () => {
     return web.ServerInfo().then(res => {
       const serverInfo = {
         version: res.MinioVersion,
-        memory: res.MinioMemory,
         platform: res.MinioPlatform,
         runtime: res.MinioRuntime,
         info: res.MinioGlobalInfo,

browser/app/js/browser/reducer.js

@@ -19,7 +19,7 @@ import * as actionsCommon from "./actions"
 export default (
   state = {
     sidebarOpen: false,
-    storageInfo: { total: 0, free: 0 },
+    storageInfo: { total: [0], free: [0], used: [0] },
     serverInfo: {}
   },
   action

browser/app/js/browser/utils.js

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,39 +14,16 @@
  * limitations under the License.
  */
 
-package disk
+export const OPEN_ID_NONCE_KEY = 'openIDKey'
 
-import (
-	"os"
-)
+export const buildOpenIDAuthURL = (authorizationEndpoint, redirectURI, clientID, nonce) => {
+  const params = new URLSearchParams()
+  params.set("response_type", "id_token")
+  params.set("scope", "openid")
+  params.set("client_id", clientID)
+  params.set("redirect_uri", redirectURI)
+  params.set("nonce", nonce)
 
-func readFile(path string, buf []byte) (int, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return 0, err
-	}
-	defer f.Close()
-
-	n, err := f.Read(buf)
-	if err != nil {
-		return 0, err
-	}
-
-	return n, nil
+  return `${authorizationEndpoint}?${params.toString()}`
 }
 
-func writeFile(path string, data []byte) (int, error) {
-	f, err := os.Create(path)
-	if err != nil {
-		return 0, err
-	}
-	defer f.Close()
-
-	n, err := f.Write(data)
-	if err != nil {
-		return 0, err
-	}
-
-	f.Sync()
-	return n, nil
-}

browser/app/js/buckets/BucketList.js

@@ -17,14 +17,29 @@
 import React from "react"
 import { connect } from "react-redux"
 import { Scrollbars } from "react-custom-scrollbars"
+import InfiniteScroll from "react-infinite-scroller"
 import * as actionsBuckets from "./actions"
-import { getVisibleBuckets } from "./selectors"
+import { getFilteredBuckets } from "./selectors"
 import BucketContainer from "./BucketContainer"
 import web from "../web"
 import history from "../history"
 import { pathSlice } from "../utils"
 
 export class BucketList extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      page: 1
+    }
+    this.loadNextPage = this.loadNextPage.bind(this)
+  }
+  componentDidUpdate(prevProps) {
+    if (this.props.filter !== prevProps.filter) {
+      this.setState({
+        page: 1
+      })
+    }
+  }
   componentWillMount() {
     const { fetchBuckets, setBucketList, selectBucket } = this.props
     if (web.LoggedIn()) {
@@ -39,18 +54,33 @@ export class BucketList extends React.Component {
       }
     }
   }
+  loadNextPage() {
+    this.setState({
+      page: this.state.page + 1
+    })
+  }
   render() {
-    const { visibleBuckets } = this.props
+    const { filteredBuckets } = this.props
+    const visibleBuckets = filteredBuckets.slice(0, this.state.page * 100)
     return (
       <div className="fesl-inner">
         <Scrollbars
           renderTrackVertical={props => <div className="scrollbar-vertical" />}
         >
-          <ul>
-            {visibleBuckets.map(bucket => (
-              <BucketContainer key={bucket} bucket={bucket} />
-            ))}
-          </ul>
+          <InfiniteScroll
+            pageStart={0}
+            loadMore={this.loadNextPage}
+            hasMore={filteredBuckets.length > visibleBuckets.length}
+            useWindow={false}
+            element="div"
+            initialLoad={false}
+          >
+            <ul>
+              {visibleBuckets.map(bucket => (
+                <BucketContainer key={bucket} bucket={bucket} />
+              ))}
+            </ul>
+          </InfiniteScroll>
         </Scrollbars>
       </div>
     )
@@ -59,7 +89,8 @@ export class BucketList extends React.Component {
 
 const mapStateToProps = state => {
   return {
-    visibleBuckets: getVisibleBuckets(state)
+    filteredBuckets: getFilteredBuckets(state),
+    filter: state.buckets.filter
   }
 }
 

browser/app/js/buckets/__tests__/BucketList.test.js

@@ -29,13 +29,13 @@ jest.mock("../../web", () => ({
 describe("BucketList", () => {
   it("should render without crashing", () => {
     const fetchBuckets = jest.fn()
-    shallow(<BucketList visibleBuckets={[]} fetchBuckets={fetchBuckets} />)
+    shallow(<BucketList filteredBuckets={[]} fetchBuckets={fetchBuckets} />)
   })
 
   it("should call fetchBuckets before component is mounted", () => {
     const fetchBuckets = jest.fn()
     const wrapper = shallow(
-      <BucketList visibleBuckets={[]} fetchBuckets={fetchBuckets} />
+      <BucketList filteredBuckets={[]} fetchBuckets={fetchBuckets} />
     )
     expect(fetchBuckets).toHaveBeenCalled()
   })
@@ -46,7 +46,7 @@ describe("BucketList", () => {
     history.push("/bk1/pre1")
     const wrapper = shallow(
       <BucketList
-        visibleBuckets={[]}
+        filteredBuckets={[]}
         setBucketList={setBucketList}
         selectBucket={selectBucket}
       />

browser/app/js/buckets/__tests__/selectors.test.js

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-import { getVisibleBuckets, getCurrentBucket } from "../selectors"
+import { getFilteredBuckets, getCurrentBucket } from "../selectors"
 
-describe("getVisibleBuckets", () => {
+describe("getFilteredBuckets", () => {
   let state
   beforeEach(() => {
     state = {
@@ -28,11 +28,11 @@ describe("getVisibleBuckets", () => {
 
   it("should return all buckets if no filter specified", () => {
     state.buckets.filter = ""
-    expect(getVisibleBuckets(state)).toEqual(["test1", "test11", "test2"])
+    expect(getFilteredBuckets(state)).toEqual(["test1", "test11", "test2"])
   })
 
   it("should return all matching buckets if filter is specified", () => {
     state.buckets.filter = "test1"
-    expect(getVisibleBuckets(state)).toEqual(["test1", "test11"])
+    expect(getFilteredBuckets(state)).toEqual(["test1", "test11"])
   })
 })

browser/app/js/buckets/selectors.js

@@ -19,7 +19,7 @@ import { createSelector } from "reselect"
 const bucketsSelector = state => state.buckets.list
 const bucketsFilterSelector = state => state.buckets.filter
 
-export const getVisibleBuckets = createSelector(
+export const getFilteredBuckets = createSelector(
   bucketsSelector,
   bucketsFilterSelector,
   (buckets, filter) => buckets.filter(bucket => bucket.indexOf(filter) > -1)

browser/app/js/utils.js

@@ -107,3 +107,13 @@ export const getRandomSecretKey = () => {
   const base64Str = btoa(binStr)
   return base64Str.replace(/\//g, "+").substr(0, 40)
 }
+
+export const getRandomString = length => {
+  var text = ""
+  var possible =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+  for (var i = 0; i < length; i++) {
+    text += possible.charAt(Math.floor(Math.random() * possible.length))
+  }
+  return text
+}

browser/app/js/web.js

@@ -75,6 +75,16 @@ class Web {
   GetToken() {
     return storage.getItem('token')
   }
+  GetDiscoveryDoc() {
+    return this.makeCall("GetDiscoveryDoc")
+  }
+  LoginSTS(args) {
+    return this.makeCall('LoginSTS', args)
+      .then(res => {
+        storage.setItem('token', `${res.token}`)
+        return res
+      })
+  }
   ServerInfo() {
     return this.makeCall('ServerInfo')
   }
@@ -125,4 +135,4 @@ class Web {
 
 const web = new Web(`${window.location.protocol}//${window.location.host}${minioBrowserPrefix}/webrpc`);
 
-export default web;
+export default web;

browser/app/less/inc/login.less

@@ -95,6 +95,41 @@
     }
 }
 
+.openid-login {
+    margin-top: 30px;
+}
+
+.openid-btn {
+    display: inline-block;
+    color: @link-color;
+    margin-top: 30px;
+    border-width: 1px;
+    border-style: solid;
+    opacity: 0.6;
+    font-size: 14px;
+    &:hover {
+        opacity: 1;
+        cursor: pointer;
+    }
+}
+
+.or {
+    display:flex;
+    justify-content:center;
+    align-items: center;
+    color:grey;
+  }
+
+.or:after,
+.or:before {
+    content: "";
+    display: block;
+    background: grey;
+    width: 10px;
+    height: 1px;
+    margin: 0 10px;
+}
+
 /*------------------------------
     Chrome autofill fix
 -------------------------------*/
@@ -102,4 +137,4 @@ input:-webkit-autofill {
      -webkit-box-shadow:0 0 0 50px #002a37 inset !important;
      -webkit-text-fill-color: @white !important;
     caret-color: white;
-}
+}

browser/package-lock.json

@@ -896,6 +896,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -930,6 +938,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -959,6 +975,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1001,6 +1025,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1073,6 +1105,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1221,6 +1261,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1337,6 +1385,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1540,6 +1596,14 @@
         "babel-runtime": "^6.22.0",
         "core-js": "^2.4.0",
         "regenerator-runtime": "^0.10.0"
+      },
+      "dependencies": {
+        "core-js": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+          "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+          "dev": true
+        }
       }
     },
     "babel-preset-es2015": {
@@ -1653,6 +1717,13 @@
       "requires": {
         "core-js": "^2.4.0",
         "regenerator-runtime": "^0.10.0"
+      },
+      "dependencies": {
+        "core-js": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+          "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A=="
+        }
       }
     },
     "babel-template": {
@@ -1676,6 +1747,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1711,6 +1790,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1741,6 +1828,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1889,9 +1984,9 @@
       "dev": true
     },
     "bootstrap": {
-      "version": "3.3.7",
-      "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.3.7.tgz",
-      "integrity": "sha1-WjiTlFSfIzMIdaOxUGVldPip63E="
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.4.1.tgz",
+      "integrity": "sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA=="
     },
     "brace-expansion": {
       "version": "1.1.11",
@@ -2883,9 +2978,9 @@
       }
     },
     "core-js": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.4.1.tgz",
-      "integrity": "sha1-TekR5mew6ukSTjQlS1OupvxhjT4="
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.2.1.tgz",
+      "integrity": "sha512-Qa5XSVefSVPRxy2XfUC13WbvqkxhkwB3ve+pgCQveNgYzbM/UxZeu1dcOX/xr4UmfUd+muuvsaxilQzCyUurMw=="
     },
     "core-util-is": {
       "version": "1.0.2",
@@ -5326,9 +5421,9 @@
       "dev": true
     },
     "handlebars": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz",
-      "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==",
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.4.0.tgz",
+      "integrity": "sha512-xkRtOt3/3DzTKMOt3xahj2M/EqNhY988T+imYSlMgs5fVhLN2fmKVVj0LtEGmb+3UUYV5Qmm1052Mm3dIQxOvw==",
       "dev": true,
       "requires": {
         "neo-async": "^2.6.0",
@@ -7340,13 +7435,21 @@
       "integrity": "sha1-COnxMkhKLEWjCQfp3E1VZ7fxFNc="
     },
     "js-yaml": {
-      "version": "3.7.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz",
-      "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=",
+      "version": "3.13.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz",
+      "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==",
       "dev": true,
       "requires": {
         "argparse": "^1.0.7",
-        "esprima": "^2.6.0"
+        "esprima": "^4.0.0"
+      },
+      "dependencies": {
+        "esprima": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+          "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+          "dev": true
+        }
       }
     },
     "jsbn": {
@@ -8081,9 +8184,9 @@
       }
     },
     "mixin-deep": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz",
-      "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
+      "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==",
       "requires": {
         "for-in": "^1.0.2",
         "is-extendable": "^1.0.1"
@@ -8356,6 +8459,24 @@
         "prepend-http": "^1.0.0",
         "query-string": "^4.1.0",
         "sort-keys": "^1.0.0"
+      },
+      "dependencies": {
+        "query-string": {
+          "version": "4.3.4",
+          "resolved": "https://registry.npmjs.org/query-string/-/query-string-4.3.4.tgz",
+          "integrity": "sha1-u7aTucqRXCMlFbIosaArYJBD2+s=",
+          "dev": true,
+          "requires": {
+            "object-assign": "^4.1.0",
+            "strict-uri-encode": "^1.0.0"
+          }
+        },
+        "strict-uri-encode": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz",
+          "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=",
+          "dev": true
+        }
       }
     },
     "npm-path": {
@@ -10184,13 +10305,13 @@
       "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A=="
     },
     "query-string": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/query-string/-/query-string-4.3.4.tgz",
-      "integrity": "sha1-u7aTucqRXCMlFbIosaArYJBD2+s=",
-      "dev": true,
+      "version": "6.8.2",
+      "resolved": "https://registry.npmjs.org/query-string/-/query-string-6.8.2.tgz",
+      "integrity": "sha512-J3Qi8XZJXh93t2FiKyd/7Ec6GNifsjKXUsVFkSBj/kjLsDylWhnCz4NT1bkPcKotttPW+QbKGqqPH8OoI2pdqw==",
       "requires": {
-        "object-assign": "^4.1.0",
-        "strict-uri-encode": "^1.0.0"
+        "decode-uri-component": "^0.2.0",
+        "split-on-first": "^1.0.0",
+        "strict-uri-encode": "^2.0.0"
       }
     },
     "querystring": {
@@ -11648,9 +11769,9 @@
       "dev": true
     },
     "set-value": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
-      "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz",
+      "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==",
       "requires": {
         "extend-shallow": "^2.0.1",
         "is-extendable": "^0.1.1",
@@ -12039,6 +12160,11 @@
         }
       }
     },
+    "split-on-first": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz",
+      "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw=="
+    },
     "split-string": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz",
@@ -12209,10 +12335,9 @@
       "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI="
     },
     "strict-uri-encode": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz",
-      "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=",
-      "dev": true
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz",
+      "integrity": "sha1-ucczDHBChi9rFC3CdLvMWGbONUY="
     },
     "string-length": {
       "version": "2.0.0",
@@ -12382,6 +12507,18 @@
         "mkdirp": "~0.5.1",
         "sax": "~1.2.1",
         "whet.extend": "~0.9.9"
+      },
+      "dependencies": {
+        "js-yaml": {
+          "version": "3.7.0",
+          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz",
+          "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=",
+          "dev": true,
+          "requires": {
+            "argparse": "^1.0.7",
+            "esprima": "^2.6.0"
+          }
+        }
       }
     },
     "symbol-observable": {
@@ -12917,35 +13054,14 @@
       }
     },
     "union-value": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
-      "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
+      "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==",
       "requires": {
         "arr-union": "^3.1.0",
         "get-value": "^2.0.6",
         "is-extendable": "^0.1.1",
-        "set-value": "^0.4.3"
-      },
-      "dependencies": {
-        "extend-shallow": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-          "requires": {
-            "is-extendable": "^0.1.0"
-          }
-        },
-        "set-value": {
-          "version": "0.4.3",
-          "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
-          "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
-          "requires": {
-            "extend-shallow": "^2.0.1",
-            "is-extendable": "^0.1.1",
-            "is-plain-object": "^2.0.1",
-            "to-object-path": "^0.3.0"
-          }
-        }
+        "set-value": "^2.0.1"
       }
     },
     "uniq": {

browser/package.json

@@ -60,8 +60,9 @@
   },
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.10.0",
-    "bootstrap": "^3.3.6",
+    "bootstrap": "^3.4.1",
     "classnames": "^2.2.3",
+    "core-js": "^3.2.1",
     "expect": "^1.20.2",
     "glob-all": "^3.1.0",
     "history": "^4.7.2",
@@ -74,6 +75,7 @@
     "mime-db": "^1.25.0",
     "mime-types": "^2.1.13",
     "moment": "^2.24.0",
+    "query-string": "^6.8.2",
     "react": "^16.2.0",
     "react-addons-test-utils": "^0.14.8",
     "react-bootstrap": "^0.32.0",

browser/webpack.config.js

@@ -74,7 +74,8 @@ var exports = {
     proxy: {
       '/minio/webrpc': {
 	target: 'http://localhost:9000',
-	secure: false
+  secure: false,
+  headers: {'Host': "localhost:9000"}
       },
       '/minio/upload/*': {
 	target: 'http://localhost:9000',

buildscripts/checkdeps.sh

@@ -21,7 +21,7 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GO_VERSION="1.12"
+    GO_VERSION="1.13"
     OSX_VERSION="10.8"
     KNAME=$(uname -s)
     ARCH=$(uname -m)

buildscripts/cross-compile.sh

@@ -23,7 +23,6 @@ function _build() {
     export GOOS=$os
     export GOARCH=$arch
     export GO111MODULE=on
-    export GOPROXY=https://proxy.golang.org
     go build -tags kqueue -o /dev/null
 }
 

buildscripts/gateway-tests.sh

@@ -24,7 +24,7 @@ function start_minio_server()
     MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 \
                     minio --quiet --json server /data --address 127.0.0.1:24242 > server.log 2>&1 &
     server_pid=$!
-    sleep 3
+    sleep 10
 
     echo "$server_pid"
 }
@@ -35,7 +35,7 @@ function start_minio_gateway_s3()
                     minio --quiet --json gateway s3 http://127.0.0.1:24242 \
                     --address 127.0.0.1:24240 > gateway.log 2>&1 &
     gw_pid=$!
-    sleep 3
+    sleep 10
 
     echo "$gw_pid"
 }
@@ -46,9 +46,10 @@ function main()
     gw_pid="$(start_minio_gateway_s3)"
 
     SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \
-                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh aws-sdk-go \
-                   aws-sdk-java aws-sdk-php aws-sdk-ruby awscli healthcheck minio-dotnet \
-                   minio-go minio-java minio-js minio-py
+                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh \
+                   aws-sdk-go aws-sdk-java aws-sdk-php aws-sdk-ruby awscli \
+                   healthcheck mc minio-dotnet minio-go minio-java minio-py \
+                   s3cmd security
     rv=$?
 
     kill "$sr_pid"

buildscripts/go-coverage.sh

@@ -2,4 +2,4 @@
 
 set -e
 
-GOPROXY=https://proxy.golang.org GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...
+GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...

buildscripts/verify-build.sh

@@ -33,7 +33,6 @@ export ACCESS_KEY="minio"
 export SECRET_KEY="minio123"
 export ENABLE_HTTPS=0
 export GO111MODULE=on
-export GOPROXY=https://proxy.golang.org
 
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )
@@ -70,63 +69,35 @@ function start_minio_erasure_sets()
     echo "$minio_pid"
 }
 
-function start_minio_dist_erasure_sets_ipv6()
+function start_minio_zone_erasure_sets()
 {
     declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
-    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9000.log" 2>&1 &
-    minio_pids[0]=$!
-    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9001.log" 2>&1 &
-    minio_pids[1]=$!
-    "${MINIO[@]}" server --address="[::1]:9002" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9002.log" 2>&1 &
-    minio_pids[2]=$!
-    "${MINIO[@]}" server --address="[::1]:9003" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9003.log" 2>&1 &
-    minio_pids[3]=$!
-    "${MINIO[@]}" server --address="[::1]:9004" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9004.log" 2>&1 &
-    minio_pids[4]=$!
-    "${MINIO[@]}" server --address="[::1]:9005" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9005.log" 2>&1 &
-    minio_pids[5]=$!
-    "${MINIO[@]}" server --address="[::1]:9006" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9006.log" 2>&1 &
-    minio_pids[6]=$!
-    "${MINIO[@]}" server --address="[::1]:9007" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9007.log" 2>&1 &
-    minio_pids[7]=$!
-    "${MINIO[@]}" server --address="[::1]:9008" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9008.log" 2>&1 &
-    minio_pids[8]=$!
-    "${MINIO[@]}" server --address="[::1]:9009" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9009.log" 2>&1 &
-    minio_pids[9]=$!
 
-    sleep 35
+    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
+    minio_pids[0]=$!
+
+    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
+    minio_pids[1]=$!
+
+    sleep 40
     echo "${minio_pids[@]}"
 }
 
-function start_minio_dist_erasure_sets()
+function start_minio_zone_erasure_sets_ipv6()
 {
     declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
-    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
-    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
-    "${MINIO[@]}" server --address=:9002 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9002.log" 2>&1 &
-    minio_pids[2]=$!
-    "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
-    minio_pids[3]=$!
-    "${MINIO[@]}" server --address=:9004 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9004.log" 2>&1 &
-    minio_pids[4]=$!
-    "${MINIO[@]}" server --address=:9005 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9005.log" 2>&1 &
-    minio_pids[5]=$!
-    "${MINIO[@]}" server --address=:9006 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9006.log" 2>&1 &
-    minio_pids[6]=$!
-    "${MINIO[@]}" server --address=:9007 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9007.log" 2>&1 &
-    minio_pids[7]=$!
-    "${MINIO[@]}" server --address=:9008 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9008.log" 2>&1 &
-    minio_pids[8]=$!
-    "${MINIO[@]}" server --address=:9009 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9009.log" 2>&1 &
-    minio_pids[9]=$!
 
-    sleep 30
+    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
+    minio_pids[0]=$!
+
+    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
+    minio_pids[1]=$!
+
+    sleep 40
     echo "${minio_pids[@]}"
 }
 
@@ -144,7 +115,7 @@ function start_minio_dist_erasure()
     "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
     minio_pids[3]=$!
 
-    sleep 30
+    sleep 40
     echo "${minio_pids[@]}"
 }
 
@@ -211,9 +182,9 @@ function run_test_dist_erasure_sets_ipv6()
     return "$rv"
 }
 
-function run_test_dist_erasure_sets()
+function run_test_zone_erasure_sets()
 {
-    minio_pids=( $(start_minio_dist_erasure_sets) )
+    minio_pids=( $(start_minio_zone_erasure_sets) )
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
@@ -224,14 +195,42 @@ function run_test_dist_erasure_sets()
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
-        for i in $(seq 0 9); do
+        for i in $(seq 0 1); do
             echo "server$i log:"
-            cat "$WORK_DIR/dist-minio-900$i.log"
+            cat "$WORK_DIR/zone-minio-900$i.log"
         done
     fi
 
-    for i in $(seq 0 9); do
-        rm -f "$WORK_DIR/dist-minio-900$i.log"
+    for i in $(seq 0 1); do
+        rm -f "$WORK_DIR/zone-minio-900$i.log"
+    done
+
+    return "$rv"
+}
+
+function run_test_zone_erasure_sets_ipv6()
+{
+    minio_pids=( $(start_minio_zone_erasure_sets_ipv6) )
+
+    export SERVER_ENDPOINT="[::1]:9000"
+
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    for pid in "${minio_pids[@]}"; do
+        kill "$pid"
+    done
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        for i in $(seq 0 1); do
+            echo "server$i log:"
+            cat "$WORK_DIR/zone-minio-ipv6-900$i.log"
+        done
+    fi
+
+    for i in $(seq 0 1); do
+        rm -f "$WORK_DIR/zone-minio-ipv6-900$i.log"
     done
 
     return "$rv"
@@ -352,15 +351,15 @@ function main()
         exit 1
     fi
 
-    echo "Testing in Distributed Erasure setup as sets"
-    if ! run_test_dist_erasure_sets; then
+    echo "Testing in Distributed Eraure expanded setup"
+    if ! run_test_zone_erasure_sets; then
         echo "FAILED"
         purge "$WORK_DIR"
         exit 1
     fi
 
-    echo "Testing in Distributed Erasure setup as sets with ipv6"
-    if ! run_test_dist_erasure_sets_ipv6; then
+    echo "Testing in Distributed Erasure expanded setup with ipv6"
+    if ! run_test_zone_erasure_sets_ipv6; then
         echo "FAILED"
         purge "$WORK_DIR"
         exit 1

cmd/admin-handlers-config-kv.go

@@ -0,0 +1,482 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/cache"
+	"github.com/minio/minio/cmd/config/etcd"
+	xldap "github.com/minio/minio/cmd/config/identity/ldap"
+	"github.com/minio/minio/cmd/config/identity/openid"
+	"github.com/minio/minio/cmd/config/policy/opa"
+	"github.com/minio/minio/cmd/config/storageclass"
+	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/minio/cmd/logger"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return nil
+	}
+
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return nil
+	}
+
+	return objectAPI
+}
+
+// DelConfigKVHandler - DELETE /minio/admin/v2/del-config-kv
+func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DelConfigKVHandler")
+
+	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err, logger.Application)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	cfg, err := readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = cfg.DelFrom(bytes.NewReader(kvBytes)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+// SetConfigKVHandler - PUT /minio/admin/v2/set-config-kv
+func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetConfigKVHandler")
+
+	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err, logger.Application)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	cfg, err := readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = validateConfig(cfg); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}
+
+	// Update the actual server config on disk.
+	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Write to the config input KV to history.
+	if err = saveServerConfigHistory(ctx, objectAPI, kvBytes); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Make sure to write backend is encrypted
+	if globalConfigEncrypted {
+		saveConfig(context.Background(), objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
+	}
+
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetConfigKVHandler - GET /minio/admin/v2/get-config-kv?key={key}
+func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetConfigKVHandler")
+
+	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	cfg := globalServerConfig
+	if globalSafeMode {
+		var err error
+		cfg, err = getValidConfig(objectAPI)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	vars := mux.Vars(r)
+	var buf = &bytes.Buffer{}
+	cw := config.NewConfigWriteTo(cfg, vars["key"])
+	if _, err := cw.WriteTo(buf); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	econfigData, err := madmin.EncryptData(password, buf.Bytes())
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ClearConfigHistoryKVHandler")
+
+	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	restoreID := vars["restoreId"]
+	if restoreID == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+	if restoreID == "all" {
+		chEntries, err := listServerConfigHistory(ctx, objectAPI, false, -1)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		for _, chEntry := range chEntries {
+			if err = delServerConfigHistory(ctx, objectAPI, chEntry.RestoreID); err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+		}
+	} else {
+		if err := delServerConfigHistory(ctx, objectAPI, restoreID); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+}
+
+// RestoreConfigHistoryKVHandler - restores a config with KV settings for the given KV id.
+func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RestoreConfigHistoryKVHandler")
+
+	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	restoreID := vars["restoreId"]
+	if restoreID == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	kvBytes, err := readServerConfigHistory(ctx, objectAPI, restoreID)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	cfg, err := readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = validateConfig(cfg); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}
+
+	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	delServerConfigHistory(ctx, objectAPI, restoreID)
+}
+
+// ListConfigHistoryKVHandler - lists all the KV ids.
+func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListConfigHistoryKVHandler")
+
+	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	count, err := strconv.Atoi(vars["count"])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	chEntries, err := listServerConfigHistory(ctx, objectAPI, true, count)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(chEntries)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// HelpConfigKVHandler - GET /minio/admin/v2/help-config-kv?subSys={subSys}&key={key}
+func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "HelpConfigKVHandler")
+
+	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+
+	subSys := vars["subSys"]
+	key := vars["key"]
+
+	_, envOnly := r.URL.Query()["env"]
+
+	rd, err := GetHelp(subSys, key, envOnly)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	json.NewEncoder(w).Encode(rd)
+	w.(http.Flusher).Flush()
+}
+
+// SetConfigHandler - PUT /minio/admin/v2/config
+func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetConfigHandler")
+
+	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err, logger.Application)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	cfg := newServerConfig()
+	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = validateConfig(cfg); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}
+
+	// Update the actual server config on disk.
+	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Write to the config input KV to history.
+	if err = saveServerConfigHistory(ctx, objectAPI, kvBytes); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Make sure to write backend is encrypted
+	if globalConfigEncrypted {
+		saveConfig(context.Background(), objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
+	}
+
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetConfigHandler - GET /minio/admin/v2/config
+// Get config.json of this minio setup.
+func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetConfigHandler")
+
+	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	cfg, err := readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	var s strings.Builder
+	hkvs := config.HelpSubSysMap[""]
+	var count int
+	for _, hkv := range hkvs {
+		count += len(cfg[hkv.Key])
+	}
+	for _, hkv := range hkvs {
+		v := cfg[hkv.Key]
+		for target, kv := range v {
+			off := kv.Get(config.Enable) == config.EnableOff
+			switch hkv.Key {
+			case config.EtcdSubSys:
+				off = !etcd.Enabled(kv)
+			case config.CacheSubSys:
+				off = !cache.Enabled(kv)
+			case config.StorageClassSubSys:
+				off = !storageclass.Enabled(kv)
+			case config.KmsVaultSubSys:
+				off = !crypto.EnabledVault(kv)
+			case config.KmsKesSubSys:
+				off = !crypto.EnabledKes(kv)
+			case config.PolicyOPASubSys:
+				off = !opa.Enabled(kv)
+			case config.IdentityOpenIDSubSys:
+				off = !openid.Enabled(kv)
+			case config.IdentityLDAPSubSys:
+				off = !xldap.Enabled(kv)
+			}
+			if off {
+				s.WriteString(config.KvComment)
+				s.WriteString(config.KvSpaceSeparator)
+			}
+			s.WriteString(hkv.Key)
+			if target != config.Default {
+				s.WriteString(config.SubSystemSeparator)
+				s.WriteString(target)
+			}
+			s.WriteString(config.KvSpaceSeparator)
+			s.WriteString(kv.String())
+			count--
+			if count > 0 {
+				s.WriteString(config.KvNewline)
+			}
+		}
+	}
+
+	password := globalActiveCred.SecretKey
+	econfigData, err := madmin.EncryptData(password, []byte(s.String()))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}

cmd/admin-handlers-users.go

@@ -0,0 +1,555 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.Request, action iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
+	var cred auth.Credentials
+	var adminAPIErr APIErrorCode
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return nil, cred
+	}
+
+	// Validate request signature.
+	cred, adminAPIErr = checkAdminRequestAuthType(ctx, r, action, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return nil, cred
+	}
+
+	return objectAPI, cred
+}
+
+// RemoveUser - DELETE /minio/admin/v2/remove-user?accessKey=<access_key>
+func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RemoveUser")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+
+	ok, err := globalIAMSys.IsTempUser(accessKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if ok {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+		return
+	}
+
+	if err := globalIAMSys.DeleteUser(accessKey); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to delete user.
+	for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// ListUsers - GET /minio/admin/v2/list-users
+func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListUsers")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	allCredentials, err := globalIAMSys.ListUsers()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(allCredentials)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// GetUserInfo - GET /minio/admin/v2/user-info
+func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetUserInfo")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	name := vars["accessKey"]
+
+	userInfo, err := globalIAMSys.GetUserInfo(name)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(userInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, data)
+}
+
+// UpdateGroupMembers - PUT /minio/admin/v2/update-group-members
+func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "UpdateGroupMembers")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	defer r.Body.Close()
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	var updReq madmin.GroupAddRemove
+	err = json.Unmarshal(data, &updReq)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	if updReq.IsRemove {
+		err = globalIAMSys.RemoveUsersFromGroup(updReq.Group, updReq.Members)
+	} else {
+		err = globalIAMSys.AddUsersToGroup(updReq.Group, updReq.Members)
+	}
+
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to load group.
+	for _, nerr := range globalNotificationSys.LoadGroup(updReq.Group) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// GetGroup - /minio/admin/v2/group?group=mygroup1
+func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetGroup")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetGroupAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	group := vars["group"]
+
+	gdesc, err := globalIAMSys.GetGroupDescription(group)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	body, err := json.Marshal(gdesc)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// ListGroups - GET /minio/admin/v2/groups
+func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListGroups")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	groups, err := globalIAMSys.ListGroups()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	body, err := json.Marshal(groups)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// SetGroupStatus - PUT /minio/admin/v2/set-group-status?group=mygroup1&status=enabled
+func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetGroupStatus")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	group := vars["group"]
+	status := vars["status"]
+
+	var err error
+	if status == statusEnabled {
+		err = globalIAMSys.SetGroupStatus(group, true)
+	} else if status == statusDisabled {
+		err = globalIAMSys.SetGroupStatus(group, false)
+	} else {
+		err = errInvalidArgument
+	}
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to reload user.
+	for _, nerr := range globalNotificationSys.LoadGroup(group) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// SetUserStatus - PUT /minio/admin/v2/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
+func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetUserStatus")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+	status := vars["status"]
+
+	// Custom IAM policies not allowed for admin user.
+	if accessKey == globalActiveCred.AccessKey {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	if err := globalIAMSys.SetUserStatus(accessKey, madmin.AccountStatus(status)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to reload user.
+	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// AddUser - PUT /minio/admin/v2/add-user?accessKey=<access_key>
+func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddUser")
+
+	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+
+	// Custom IAM policies not allowed for admin user.
+	if accessKey == globalActiveCred.AccessKey {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := cred.SecretKey
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	var uinfo madmin.UserInfo
+	if err = json.Unmarshal(configBytes, &uinfo); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other Minio peers to reload user
+	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// InfoCannedPolicy - GET /minio/admin/v2/info-canned-policy?name={policyName}
+func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "InfoCannedPolicy")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	data, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.Write(data)
+	w.(http.Flusher).Flush()
+}
+
+// ListCannedPolicies - GET /minio/admin/v2/list-canned-policies
+func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListCannedPolicies")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	policies, err := globalIAMSys.ListPolicies()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(policies); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// RemoveCannedPolicy - DELETE /minio/admin/v2/remove-canned-policy?name=<policy_name>
+func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RemoveCannedPolicy")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars["name"]
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if err := globalIAMSys.DeletePolicy(policyName); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to delete policy
+	for _, nerr := range globalNotificationSys.DeletePolicy(policyName) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// AddCannedPolicy - PUT /minio/admin/v2/add-canned-policy?name=<policy_name>
+func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddCannedPolicy")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars["name"]
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	// Error out if Content-Length is missing.
+	if r.ContentLength <= 0 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
+		return
+	}
+
+	// Error out if Content-Length is beyond allowed size.
+	if r.ContentLength > maxBucketPolicySize {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL)
+		return
+	}
+
+	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
+		return
+	}
+
+	// Version in policy must not be empty
+	if iamPolicy.Version == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
+		return
+	}
+
+	if err = globalIAMSys.SetPolicy(policyName, *iamPolicy); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to reload policy
+	for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// SetPolicyForUserOrGroup - PUT /minio/admin/v2/set-policy?policy=xxx&user-or-group=?[&is-group]
+func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars["policyName"]
+	entityName := vars["userOrGroup"]
+	isGroup := vars["isGroup"] == "true"
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if !isGroup {
+		ok, err := globalIAMSys.IsTempUser(entityName)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if ok {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+			return
+		}
+	}
+
+	if err := globalIAMSys.PolicyDBSet(entityName, policyName, isGroup); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to reload policy
+	for _, nerr := range globalNotificationSys.LoadPolicyMapping(entityName, isGroup) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}

cmd/admin-handlers_test.go

@@ -25,7 +25,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"strings"
 	"sync"
 	"testing"
 
@@ -34,217 +33,6 @@ import (
 	"github.com/minio/minio/pkg/madmin"
 )
 
-var (
-	configJSON = []byte(`{
-  "version": "33",
-  "credential": {
-    "accessKey": "minio",
-    "secretKey": "minio123"
-  },
-  "region": "us-east-1",
-  "worm": "off",
-  "storageclass": {
-    "standard": "",
-    "rrs": ""
-  },
-  "cache": {
-    "drives": [],
-    "expiry": 90,
-    "maxuse": 80,
-    "exclude": []
-  },
-  "kms": {
-    "vault": {
-      "endpoint": "",
-      "auth": {
-        "type": "",
-        "approle": {
-          "id": "",
-          "secret": ""
-        }
-      },
-      "key-id": {
-        "name": "",
-        "version": 0
-      }
-    }
-  },
-  "notify": {
-    "amqp": {
-      "1": {
-        "enable": false,
-        "url": "",
-        "exchange": "",
-        "routingKey": "",
-        "exchangeType": "",
-        "deliveryMode": 0,
-        "mandatory": false,
-        "immediate": false,
-        "durable": false,
-        "internal": false,
-        "noWait": false,
-        "autoDeleted": false,
-        "queueDir": "",
-        "queueLimit": 0
-      }
-    },
-    "elasticsearch": {
-      "1": {
-        "enable": false,
-        "format": "namespace",
-        "url": "",
-        "index": "",
-        "queueDir": "",
-        "queueLimit": 0
-      }
-    },
-    "kafka": {
-      "1": {
-        "enable": false,
-        "brokers": null,
-        "topic": "",
-        "queueDir": "",
-        "queueLimit": 0,
-        "tls": {
-          "enable": false,
-          "skipVerify": false,
-          "clientAuth": 0
-        },
-        "sasl": {
-          "enable": false,
-          "username": "",
-          "password": ""
-        }
-      }
-    },
-    "mqtt": {
-      "1": {
-        "enable": false,
-        "broker": "",
-        "topic": "",
-        "qos": 0,
-        "username": "",
-        "password": "",
-        "reconnectInterval": 0,
-	"keepAliveInterval": 0,
-	"queueDir": "",
-        "queueLimit": 0
-      }
-    },
-    "mysql": {
-      "1": {
-        "enable": false,
-        "format": "namespace",
-        "dsnString": "",
-        "table": "",
-        "host": "",
-        "port": "",
-        "user": "",
-        "password": "",
-        "database": "",
-        "queueDir": "",
-        "queueLimit": 0
-      }
-    },
-    "nats": {
-      "1": {
-        "enable": false,
-        "address": "",
-        "subject": "",
-        "username": "",
-        "password": "",
-        "token": "",
-        "secure": false,
-        "pingInterval": 0,
-        "queueDir": "",
-        "queueLimit": 0,
-        "streaming": {
-          "enable": false,
-          "clusterID": "",
-          "async": false,
-          "maxPubAcksInflight": 0
-        }
-      }
-	},
-    "nsq": {
-      "1": {
-        "enable": false,
-        "nsqdAddress": "",
-        "topic": "",
-        "tls": {
-			"enable": false,
-			"skipVerify": false
-		},
-        "queueDir": "",
-        "queueLimit": 0
-      }
-    },
-    "postgresql": {
-      "1": {
-        "enable": false,
-        "format": "namespace",
-        "connectionString": "",
-        "table": "",
-        "host": "",
-        "port": "",
-        "user": "",
-        "password": "",
-        "database": "",
-        "queueDir": "",
-        "queueLimit": 0
-      }
-    },
-    "redis": {
-      "1": {
-        "enable": false,
-        "format": "namespace",
-        "address": "",
-        "password": "",
-        "key": "",
-        "queueDir": "",
-        "queueLimit": 0
-      }
-    },
-    "webhook": {
-      "1": {
-        "enable": false,
-        "endpoint": "",
-        "queueDir": "",
-        "queueLimit": 0
-      }
-    }
-  },
-  "logger": {
-    "console": {
-      "enabled": true
-    },
-    "http": {
-      "1": {
-        "enabled": false,
-        "endpoint": "https://username:password@example.com/api"
-      }
-    }
-  },
-  "compress": {
-    "enabled": false,
-    "extensions":[".txt",".log",".csv",".json"],
-    "mime-types":["text/csv","text/plain","application/json"]
-  },
-  "openid": {
-    "jwks": {
-      "url": ""
-    }
-  },
-  "policy": {
-    "opa": {
-      "url": "",
-      "authToken": ""
-    }
-  }
-}
-`)
-)
-
 // adminXLTestBed - encapsulates subsystems that need to be setup for
 // admin-handler unit tests.
 type adminXLTestBed struct {
@@ -273,16 +61,12 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// Initialize boot time
 	globalBootTime = UTCNow()
 
-	globalEndpoints = mustGetNewEndpointList(xlDirs...)
+	globalEndpoints = mustGetZoneEndpoints(xlDirs...)
 
 	// Set globalIsXL to indicate that the setup uses an erasure
 	// code backend.
 	globalIsXL = true
 
-	// initialize NSLock.
-	isDistXL := false
-	initNSLock(isDistXL)
-
 	// Init global heal state
 	if globalIsXL {
 		globalAllHealState = initHealState()
@@ -293,11 +77,16 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	globalIAMSys = NewIAMSys()
 	globalIAMSys.Init(objLayer)
 
-	globalPolicySys = NewPolicySys()
-	globalPolicySys.Init(objLayer)
+	buckets, err := objLayer.ListBuckets(context.Background())
+	if err != nil {
+		return nil, err
+	}
 
-	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
-	globalNotificationSys.Init(objLayer)
+	globalPolicySys = NewPolicySys()
+	globalPolicySys.Init(buckets, objLayer)
+
+	globalNotificationSys = NewNotificationSys(globalEndpoints)
+	globalNotificationSys.Init(buckets, objLayer)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
@@ -324,8 +113,8 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) {
 	if err != nil {
 		return nil, nil, err
 	}
-	endpoints := mustGetNewEndpointList(xlDirs...)
-	format, err := waitForFormatXL(context.Background(), true, endpoints, 1, 16)
+	endpoints := mustGetNewEndpoints(xlDirs...)
+	format, err := waitForFormatXL(true, endpoints, 1, 16, "")
 	if err != nil {
 		removeRoots(xlDirs)
 		return nil, nil, err
@@ -390,7 +179,7 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, error) {
 	queryVal := url.Values{}
 	queryVal.Set("action", string(cmd.toServiceAction()))
-	resource := "/minio/admin/v1/service?" + queryVal.Encode()
+	resource := adminPathPrefix + adminAPIVersionPrefix + "/service?" + queryVal.Encode()
 	req, err := newTestRequest(http.MethodPost, resource, 0, nil)
 	if err != nil {
 		return nil, err
@@ -429,7 +218,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 			testServiceSignalReceiver(cmd, t)
 		}()
 	}
-	credentials := globalServerConfig.GetCredential()
+	credentials := globalActiveCred
 
 	req, err := getServiceCmdRequest(cmd, credentials)
 	if err != nil {
@@ -459,13 +248,13 @@ func buildAdminRequest(queryVal url.Values, method, path string,
 	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {
 
 	req, err := newTestRequest(method,
-		"/minio/admin/v1"+path+"?"+queryVal.Encode(),
+		adminPathPrefix+adminAPIVersionPrefix+path+"?"+queryVal.Encode(),
 		contentLength, bodySeeker)
 	if err != nil {
 		return nil, err
 	}
 
-	cred := globalServerConfig.GetCredential()
+	cred := globalActiveCred
 	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 	if err != nil {
 		return nil, err
@@ -474,106 +263,6 @@ func buildAdminRequest(queryVal url.Values, method, path string,
 	return req, nil
 }
 
-// TestGetConfigHandler - test for GetConfigHandler.
-func TestGetConfigHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// Initialize admin peers to make admin RPC calls.
-	globalMinioAddr = "127.0.0.1:9000"
-
-	// Prepare query params for get-config mgmt REST API.
-	queryVal := url.Values{}
-	queryVal.Set("config", "")
-
-	req, err := buildAdminRequest(queryVal, http.MethodGet, "/config", 0, nil)
-	if err != nil {
-		t.Fatalf("Failed to construct get-config object request - %v", err)
-	}
-
-	rec := httptest.NewRecorder()
-	adminTestBed.router.ServeHTTP(rec, req)
-	if rec.Code != http.StatusOK {
-		t.Errorf("Expected to succeed but failed with %d", rec.Code)
-	}
-
-}
-
-// TestSetConfigHandler - test for SetConfigHandler.
-func TestSetConfigHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// Initialize admin peers to make admin RPC calls.
-	globalMinioAddr = "127.0.0.1:9000"
-
-	// Prepare query params for set-config mgmt REST API.
-	queryVal := url.Values{}
-	queryVal.Set("config", "")
-
-	password := globalServerConfig.GetCredential().SecretKey
-	econfigJSON, err := madmin.EncryptData(password, configJSON)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
-		int64(len(econfigJSON)), bytes.NewReader(econfigJSON))
-	if err != nil {
-		t.Fatalf("Failed to construct set-config object request - %v", err)
-	}
-
-	rec := httptest.NewRecorder()
-	adminTestBed.router.ServeHTTP(rec, req)
-	if rec.Code != http.StatusOK {
-		t.Errorf("Expected to succeed but failed with %d, body: %s", rec.Code, rec.Body)
-	}
-
-	// Check that a very large config file returns an error.
-	{
-		// Make a large enough config string
-		invalidCfg := []byte(strings.Repeat("A", maxEConfigJSONSize+1))
-		req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
-			int64(len(invalidCfg)), bytes.NewReader(invalidCfg))
-		if err != nil {
-			t.Fatalf("Failed to construct set-config object request - %v", err)
-		}
-
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		respBody := rec.Body.String()
-		if rec.Code != http.StatusBadRequest ||
-			!strings.Contains(respBody, "Configuration data provided exceeds the allowed maximum of") {
-			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
-		}
-	}
-
-	// Check that a config with duplicate keys in an object return
-	// error.
-	{
-		invalidCfg := append(econfigJSON[:len(econfigJSON)-1], []byte(`, "version": "15"}`)...)
-		req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
-			int64(len(invalidCfg)), bytes.NewReader(invalidCfg))
-		if err != nil {
-			t.Fatalf("Failed to construct set-config object request - %v", err)
-		}
-
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		respBody := rec.Body.String()
-		if rec.Code != http.StatusBadRequest ||
-			!strings.Contains(respBody, "JSON configuration provided is of incorrect format") {
-			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
-		}
-	}
-}
-
 func TestAdminServerInfo(t *testing.T) {
 	adminTestBed, err := prepareAdminXLTestBed()
 	if err != nil {
@@ -599,23 +288,14 @@ func TestAdminServerInfo(t *testing.T) {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
 
-	results := []ServerInfo{}
+	results := madmin.InfoMessage{}
 	err = json.NewDecoder(rec.Body).Decode(&results)
 	if err != nil {
 		t.Fatalf("Failed to decode set config result json %v", err)
 	}
 
-	if len(results) == 0 {
-		t.Error("Expected at least one server info result")
-	}
-
-	for _, serverInfo := range results {
-		if serverInfo.Error != "" {
-			t.Errorf("Unexpected error = %v\n", serverInfo.Error)
-		}
-		if serverInfo.Data.Properties.Region != globalMinioDefaultRegion {
-			t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, serverInfo.Data.Properties.Region)
-		}
+	if results.Region != globalMinioDefaultRegion {
+		t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, results.Region)
 	}
 }
 
@@ -650,3 +330,60 @@ func TestToAdminAPIErrCode(t *testing.T) {
 		}
 	}
 }
+
+func TestExtractHealInitParams(t *testing.T) {
+	mkParams := func(clientToken string, forceStart, forceStop bool) url.Values {
+		v := url.Values{}
+		if clientToken != "" {
+			v.Add(string(mgmtClientToken), clientToken)
+		}
+		if forceStart {
+			v.Add(string(mgmtForceStart), "")
+		}
+		if forceStop {
+			v.Add(string(mgmtForceStop), "")
+		}
+		return v
+	}
+	qParmsArr := []url.Values{
+		// Invalid cases
+		mkParams("", true, true),
+		mkParams("111", true, true),
+		mkParams("111", true, false),
+		mkParams("111", false, true),
+		// Valid cases follow
+		mkParams("", true, false),
+		mkParams("", false, true),
+		mkParams("", false, false),
+		mkParams("111", false, false),
+	}
+	varsArr := []map[string]string{
+		// Invalid cases
+		{string(mgmtPrefix): "objprefix"},
+		// Valid cases
+		{},
+		{string(mgmtBucket): "bucket"},
+		{string(mgmtBucket): "bucket", string(mgmtPrefix): "objprefix"},
+	}
+
+	// Body is always valid - we do not test JSON decoding.
+	body := `{"recursive": false, "dryRun": true, "remove": false, "scanMode": 0}`
+
+	// Test all combinations!
+	for pIdx, parms := range qParmsArr {
+		for vIdx, vars := range varsArr {
+			_, err := extractHealInitParams(vars, parms, bytes.NewBuffer([]byte(body)))
+			isErrCase := false
+			if pIdx < 4 || vIdx < 1 {
+				isErrCase = true
+			}
+
+			if err != ErrNone && !isErrCase {
+				t.Errorf("Got unexpected error: %v %v %v", pIdx, vIdx, err)
+			} else if err == ErrNone && isErrCase {
+				t.Errorf("Got no error but expected one: %v %v", pIdx, vIdx)
+			}
+		}
+	}
+
+}

cmd/admin-heal-ops.go

@@ -53,6 +53,10 @@ const (
 	// time-duration to keep heal sequence state after it
 	// completes.
 	keepHealSeqStateDuration = time.Minute * 10
+
+	// nopHeal is a no operating healing action to
+	// wait for the current healing operation to finish
+	nopHeal = ""
 )
 
 var (
@@ -548,7 +552,7 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 	var respCh = make(chan healResult)
 	defer close(respCh)
 	// Send heal request
-	globalBackgroundHealing.queueHealTask(healTask{path: path, responseCh: respCh, opts: h.settings})
+	globalBackgroundHealRoutine.queueHealTask(healTask{path: path, responseCh: respCh, opts: h.settings})
 	// Wait for answer and push result to the client
 	res := <-respCh
 	if !h.reportProgress {
@@ -583,10 +587,17 @@ func (h *healSequence) healItemsFromSourceCh() error {
 		logger.LogIf(h.ctx, err)
 	}
 
+	// Start healing the background ops prefix.
+	if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
+		logger.LogIf(h.ctx, err)
+	}
+
 	for path := range h.sourceCh {
 
 		var itemType madmin.HealItemType
 		switch {
+		case path == nopHeal:
+			continue
 		case path == SlashSeparator:
 			itemType = madmin.HealItemMetadata
 		case !strings.Contains(path, SlashSeparator):
@@ -607,9 +618,7 @@ func (h *healSequence) healItemsFromSourceCh() error {
 }
 
 func (h *healSequence) healFromSourceCh() {
-	if err := h.healItemsFromSourceCh(); err != nil {
-		h.traverseAndHealDoneCh <- err
-	}
+	h.healItemsFromSourceCh()
 	close(h.traverseAndHealDoneCh)
 }
 
@@ -629,6 +638,11 @@ func (h *healSequence) healItems() error {
 		return err
 	}
 
+	// Start healing the background ops prefix.
+	if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
+		logger.LogIf(h.ctx, err)
+	}
+
 	// Heal buckets and objects
 	return h.healBuckets()
 }
@@ -656,7 +670,7 @@ func (h *healSequence) traverseAndHeal() {
 func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 	return func() error {
 		// Get current object layer instance.
-		objectAPI := newObjectLayerFn()
+		objectAPI := newObjectLayerWithoutSafeModeFn()
 		if objectAPI == nil {
 			return errServerNotInitialized
 		}
@@ -688,7 +702,7 @@ func (h *healSequence) healDiskFormat() error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -708,7 +722,7 @@ func (h *healSequence) healBuckets() error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -730,7 +744,7 @@ func (h *healSequence) healBuckets() error {
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(bucket string) error {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -767,7 +781,7 @@ func (h *healSequence) healObject(bucket, object string) error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}

cmd/admin-router.go

@@ -20,121 +20,147 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 const (
-	adminAPIPathPrefix = "/minio/admin"
+	adminPathPrefix       = minioReservedBucketPath + "/admin"
+	adminAPIVersion       = madmin.AdminAPIVersion
+	adminAPIVersionPrefix = SlashSeparator + madmin.AdminAPIVersion
 )
 
 // adminAPIHandlers provides HTTP handlers for MinIO admin API.
-type adminAPIHandlers struct {
-}
+type adminAPIHandlers struct{}
 
 // registerAdminRouter - Add handler functions for each service REST API routes.
 func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) {
 
 	adminAPI := adminAPIHandlers{}
 	// Admin router
-	adminRouter := router.PathPrefix(adminAPIPathPrefix).Subrouter()
-
-	// Version handler
-	adminV1Router := adminRouter.PathPrefix("/v1").Subrouter()
+	adminRouter := router.PathPrefix(adminPathPrefix).Subrouter()
 
 	/// Service operations
 
 	// Restart and stop MinIO service.
-	adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
+	adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix+"/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
 	// Update MinIO servers.
-	adminV1Router.Methods(http.MethodPost).Path("/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
+	adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix+"/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
 
 	// Info operations
-	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
+	// Harware Info operations
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/hardware").HandlerFunc(httpTraceAll(adminAPI.ServerHardwareInfoHandler)).Queries("hwType", "{hwType:.*}")
+
+	// StorageInfo operations
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/storageinfo").HandlerFunc(httpTraceAll(adminAPI.StorageInfoHandler))
+	// DataUsageInfo operations
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
 
 	if globalIsDistXL || globalIsXL {
 		/// Heal operations
 
 		// Heal processing endpoint.
-		adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
-		adminV1Router.Methods(http.MethodPost).Path("/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
 
 		/// Health operations
 
 	}
 	// Performance command - return performance details based on input type
-	adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
 
 	// Profiling operations
-	adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
+	adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix+"/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
 		Queries("profilerType", "{profilerType:.*}")
-	adminV1Router.Methods(http.MethodGet).Path("/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
+
+	// Config KV operations.
+	if enableConfigOps {
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/get-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKVHandler)).Queries("key", "{key:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/set-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKVHandler))
+		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix + "/del-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.DelConfigKVHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)).Queries("count", "{count:[0-9]+}")
+		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/clear-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/restore-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
+	}
 
 	/// Config operations
 	if enableConfigOps {
 		// Get config
-		adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 		// Set config
-		adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
-
-		// Get config keys/values
-		adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
-		// Set config keys/values
-		adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
 	}
 
 	if enableIAMOps {
 		// -- IAM APIs --
 
 		// Add policy IAM
-		adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name",
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name",
 			"{name:.*}")
 
 		// Add user IAM
-		adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
-		adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
 			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
+		// Info policy IAM
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
+
 		// Remove policy IAM
-		adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
 
 		// Set user or group policy
-		adminV1Router.Methods(http.MethodPut).Path("/set-user-or-group-policy").
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-user-or-group-policy").
 			HandlerFunc(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup)).
 			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
 
 		// Remove user IAM
-		adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
 
 		// List users
-		adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
 
 		// User info
-		adminV1Router.Methods(http.MethodGet).Path("/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
 
 		// Add/Remove members from group
-		adminV1Router.Methods(http.MethodPut).Path("/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
 
 		// Get Group
-		adminV1Router.Methods(http.MethodGet).Path("/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
 
 		// List Groups
-		adminV1Router.Methods(http.MethodGet).Path("/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
 
 		// Set Group Status
-		adminV1Router.Methods(http.MethodPut).Path("/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
 
 		// List policies
-		adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
 	}
 
 	// -- Top APIs --
 	// Top locks
-	adminV1Router.Methods(http.MethodGet).Path("/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+	if globalIsDistXL {
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+	}
 
 	// HTTP Trace
-	adminV1Router.Methods(http.MethodGet).Path("/trace").HandlerFunc(adminAPI.TraceHandler)
-	// If none of the routes match, return error.
-	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/trace").HandlerFunc(adminAPI.TraceHandler)
+
+	// Console Logs
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/log").HandlerFunc(httpTraceAll(adminAPI.ConsoleLogHandler))
+
+	// -- KMS APIs --
+	//
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
+
+	// If none of the routes match add default error handler routes
+	adminRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))
+	adminRouter.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))
 }

cmd/admin-server-info.go

@@ -0,0 +1,261 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/cpu"
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/mem"
+
+	cpuhw "github.com/shirou/gopsutil/cpu"
+)
+
+// getLocalMemUsage - returns ServerMemUsageInfo for all zones, endpoints.
+func getLocalMemUsage(endpointZones EndpointZones, r *http.Request) ServerMemUsageInfo {
+	var memUsages []mem.Usage
+	var historicUsages []mem.Usage
+	seenHosts := set.NewStringSet()
+	for _, ep := range endpointZones {
+		for _, endpoint := range ep.Endpoints {
+			if seenHosts.Contains(endpoint.Host) {
+				continue
+			}
+			seenHosts.Add(endpoint.Host)
+
+			// Only proceed for local endpoints
+			if endpoint.IsLocal {
+				memUsages = append(memUsages, mem.GetUsage())
+				historicUsages = append(historicUsages, mem.GetHistoricUsage())
+			}
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpointZones)
+	}
+	return ServerMemUsageInfo{
+		Addr:          addr,
+		Usage:         memUsages,
+		HistoricUsage: historicUsages,
+	}
+}
+
+// getLocalCPULoad - returns ServerCPULoadInfo for all zones, endpoints.
+func getLocalCPULoad(endpointZones EndpointZones, r *http.Request) ServerCPULoadInfo {
+	var cpuLoads []cpu.Load
+	var historicLoads []cpu.Load
+	seenHosts := set.NewStringSet()
+	for _, ep := range endpointZones {
+		for _, endpoint := range ep.Endpoints {
+			if seenHosts.Contains(endpoint.Host) {
+				continue
+			}
+			seenHosts.Add(endpoint.Host)
+
+			// Only proceed for local endpoints
+			if endpoint.IsLocal {
+				cpuLoads = append(cpuLoads, cpu.GetLoad())
+				historicLoads = append(historicLoads, cpu.GetHistoricLoad())
+			}
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpointZones)
+	}
+	return ServerCPULoadInfo{
+		Addr:         addr,
+		Load:         cpuLoads,
+		HistoricLoad: historicLoads,
+	}
+}
+
+// getLocalDrivesPerf - returns ServerDrivesPerfInfo for all zones, endpoints.
+func getLocalDrivesPerf(endpointZones EndpointZones, size int64, r *http.Request) madmin.ServerDrivesPerfInfo {
+	var dps []disk.Performance
+	for _, ep := range endpointZones {
+		for _, endpoint := range ep.Endpoints {
+			// Only proceed for local endpoints
+			if endpoint.IsLocal {
+				if _, err := os.Stat(endpoint.Path); err != nil {
+					// Since this drive is not available, add relevant details and proceed
+					dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
+					continue
+				}
+				dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()), size)
+				dp.Path = endpoint.Path
+				dps = append(dps, dp)
+			}
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpointZones)
+	}
+	return madmin.ServerDrivesPerfInfo{
+		Addr: addr,
+		Perf: dps,
+	}
+}
+
+// getLocalCPUInfo - returns ServerCPUHardwareInfo for all zones, endpoints.
+func getLocalCPUInfo(endpointZones EndpointZones, r *http.Request) madmin.ServerCPUHardwareInfo {
+	var cpuHardwares []cpuhw.InfoStat
+	seenHosts := set.NewStringSet()
+	for _, ep := range endpointZones {
+		for _, endpoint := range ep.Endpoints {
+			if seenHosts.Contains(endpoint.Host) {
+				continue
+			}
+			// Add to the list of visited hosts
+			seenHosts.Add(endpoint.Host)
+			// Only proceed for local endpoints
+			if endpoint.IsLocal {
+				cpuHardware, err := cpuhw.Info()
+				if err != nil {
+					return madmin.ServerCPUHardwareInfo{
+						Error: err.Error(),
+					}
+				}
+				cpuHardwares = append(cpuHardwares, cpuHardware...)
+			}
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpointZones)
+	}
+
+	return madmin.ServerCPUHardwareInfo{
+		Addr:    addr,
+		CPUInfo: cpuHardwares,
+	}
+}
+
+// getLocalNetworkInfo - returns ServerNetworkHardwareInfo for all zones, endpoints.
+func getLocalNetworkInfo(endpointZones EndpointZones, r *http.Request) madmin.ServerNetworkHardwareInfo {
+	var networkHardwares []net.Interface
+	seenHosts := set.NewStringSet()
+	for _, ep := range endpointZones {
+		for _, endpoint := range ep.Endpoints {
+			if seenHosts.Contains(endpoint.Host) {
+				continue
+			}
+			// Add to the list of visited hosts
+			seenHosts.Add(endpoint.Host)
+			// Only proceed for local endpoints
+			if endpoint.IsLocal {
+				networkHardware, err := net.Interfaces()
+				if err != nil {
+					return madmin.ServerNetworkHardwareInfo{
+						Error: err.Error(),
+					}
+				}
+				networkHardwares = append(networkHardwares, networkHardware...)
+			}
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpointZones)
+	}
+
+	return madmin.ServerNetworkHardwareInfo{
+		Addr:        addr,
+		NetworkInfo: networkHardwares,
+	}
+}
+
+// getLocalServerProperty - returns ServerDrivesPerfInfo for only the
+// local endpoints from given list of endpoints
+func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
+	var di madmin.Disk
+	var disks []madmin.Disk
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpointZones)
+	}
+	network := make(map[string]string)
+	hosts := set.NewStringSet()
+	for _, ep := range endpointZones {
+		for _, endpoint := range ep.Endpoints {
+
+			url := strings.Replace(endpoint.URL.String(), endpoint.Path, "", -1)
+			if url == "" {
+				url = r.Host
+			}
+			hosts.Add(url)
+
+			// Only proceed for local endpoints
+			if endpoint.IsLocal {
+				url = fetchAddress(url)
+				network[url] = "online"
+				if _, err := os.Stat(endpoint.Path); err != nil {
+					continue
+				}
+
+				diInfo, _ := disk.GetInfo(endpoint.Path)
+				di.State = "ok"
+				di.DrivePath = endpoint.Path
+				di.TotalSpace = diInfo.Total
+				di.UsedSpace = diInfo.Total - diInfo.Free
+				di.Utilization = float64((diInfo.Total - diInfo.Free) / diInfo.Total * 100)
+				disks = append(disks, di)
+			}
+		}
+	}
+
+	for host := range hosts {
+		_, present := network[host]
+		if !present {
+			err := checkConnection(host)
+			host = fetchAddress(host)
+			if err != nil {
+				network[host] = "offline"
+			} else {
+				network[host] = "online"
+			}
+		}
+	}
+
+	return madmin.ServerProperties{
+		State:    "ok",
+		Endpoint: addr,
+		Uptime:   UTCNow().Unix() - globalBootTime.Unix(),
+		Version:  Version,
+		CommitID: CommitID,
+		Network:  network,
+		Disks:    disks,
+	}
+}
+
+// Replaces http and https from address
+func fetchAddress(address string) string {
+	if strings.Contains(address, "http://") {
+		address = strings.Replace(address, "http://", "", -1)
+	} else if strings.Contains(address, "https://") {
+		address = strings.Replace(address, "https://", "", -1)
+	}
+	return address
+}

cmd/api-datatypes.go

@@ -26,7 +26,7 @@ type ObjectIdentifier struct {
 }
 
 // createBucketConfiguration container for bucket configuration request from client.
-// Used for parsing the location from the request body for MakeBucketbucket.
+// Used for parsing the location from the request body for Makebucket.
 type createBucketLocationConfiguration struct {
 	XMLName  xml.Name `xml:"CreateBucketConfiguration" json:"-"`
 	Location string   `xml:"LocationConstraint"`

cmd/api-errors.go

@@ -23,15 +23,15 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"google.golang.org/api/googleapi"
 
 	minio "github.com/minio/minio-go/v6"
+	"github.com/minio/minio/cmd/config/etcd/dns"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 )
@@ -141,6 +141,12 @@ const (
 	ErrInvalidPrefixMarker
 	ErrBadRequest
 	ErrKeyTooLongError
+	ErrInvalidBucketObjectLockConfiguration
+	ErrObjectLocked
+	ErrInvalidRetentionDate
+	ErrPastObjectLockRetainDate
+	ErrUnknownWORMModeDirective
+	ErrObjectLockInvalidHeaders
 	// Add new error codes here.
 
 	// SSE-S3 related API errors
@@ -315,6 +321,7 @@ const (
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
+	ErrPostPolicyConditionInvalidFormat
 )
 
 type errorCodeMap map[APIErrorCode]APIError
@@ -719,7 +726,36 @@ var errorCodes = errorCodeMap{
 		Description:    "Duration provided in the request is invalid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
+	ErrInvalidBucketObjectLockConfiguration: {
+		Code:           "InvalidRequest",
+		Description:    "Bucket is missing ObjectLockConfiguration",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrObjectLocked: {
+		Code:           "InvalidRequest",
+		Description:    "Object is WORM protected and cannot be overwritten",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRetentionDate: {
+		Code:           "InvalidRequest",
+		Description:    "Date must be provided in ISO 8601 format",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPastObjectLockRetainDate: {
+		Code:           "InvalidRequest",
+		Description:    "the retain until date must be in the future",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrUnknownWORMModeDirective: {
+		Code:           "InvalidRequest",
+		Description:    "unknown wormMode directive",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrObjectLockInvalidHeaders: {
+		Code:           "InvalidRequest",
+		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	/// Bucket notification related errors.
 	ErrEventNotification: {
 		Code:           "InvalidArgument",
@@ -1496,6 +1532,11 @@ var errorCodes = errorCodeMap{
 		Description:    "User is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrPostPolicyConditionInvalidFormat: {
+		Code:           "PostPolicyInvalidKeyName",
+		Description:    "Invalid according to Policy: Policy Condition failed",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	// Add your error structure here.
 }
 
@@ -1563,6 +1604,14 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
 		apiErr = ErrSlowDown
+	case errInvalidRetentionDate:
+		apiErr = ErrInvalidRetentionDate
+	case errPastObjectLockRetainDate:
+		apiErr = ErrPastObjectLockRetainDate
+	case errUnknownWORMModeDirective:
+		apiErr = ErrUnknownWORMModeDirective
+	case errObjectLockInvalidHeaders:
+		apiErr = ErrObjectLockInvalidHeaders
 	}
 
 	// Compression errors
@@ -1723,6 +1772,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// their internal error types. This code is only
 		// useful with gateway implementations.
 		switch e := err.(type) {
+		case crypto.Error:
+			apiErr = APIError{
+				Code:           "XKMSInternalError",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case minio.ErrorResponse:
 			apiErr = APIError{
 				Code:           e.Code,
@@ -1741,11 +1796,11 @@ func toAPIError(ctx context.Context, err error) APIError {
 				apiErr.Code = e.Errors[0].Reason
 
 			}
-		case storage.AzureStorageServiceError:
+		case azblob.StorageError:
 			apiErr = APIError{
-				Code:           e.Code,
-				Description:    e.Message,
-				HTTPStatusCode: e.StatusCode,
+				Code:           string(e.ServiceCode()),
+				Description:    e.Error(),
+				HTTPStatusCode: e.Response().StatusCode,
 			}
 		case oss.ServiceError:
 			apiErr = APIError{
@@ -1778,7 +1833,7 @@ func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID,
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
 		Resource:   resource,
-		Region:     globalServerConfig.GetRegion(),
+		Region:     globalServerRegion,
 		RequestID:  requestID,
 		HostID:     hostID,
 	}

cmd/api-errors_test.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"context"
 	"errors"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/minio/minio/cmd/crypto"
@@ -65,6 +67,11 @@ var toAPIErrorTests = []struct {
 }
 
 func TestAPIErrCode(t *testing.T) {
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(disk)
+
+	initFSObjects(disk, t)
+
 	ctx := context.Background()
 	for i, testCase := range toAPIErrorTests {
 		errCode := toAPIErrorCode(ctx, testCase.err)

cmd/api-headers.go

@@ -40,7 +40,7 @@ func setCommonHeaders(w http.ResponseWriter) {
 	w.Header().Set(xhttp.ServerInfo, "MinIO/"+ReleaseTag)
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
-	if region := globalServerConfig.GetRegion(); region != "" {
+	if region := globalServerRegion; region != "" {
 		w.Header().Set(xhttp.AmzBucketRegion, region)
 	}
 	w.Header().Set(xhttp.AcceptRanges, "bytes")
@@ -94,7 +94,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
-		if hasPrefix(k, ReservedMetadataPrefix) {
+		if HasPrefix(k, ReservedMetadataPrefix) {
 			// Do not need to send any internal metadata
 			// values to client.
 			continue

cmd/api-resources.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"net/url"
 	"strconv"
 )
@@ -86,11 +87,19 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 	}
 
 	prefix = values.Get("prefix")
-	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
+
+	if token = values.Get("continuation-token"); token != "" {
+		decodedToken, err := base64.StdEncoding.DecodeString(token)
+		if err != nil {
+			errCode = ErrIncorrectContinuationToken
+			return
+		}
+		token = string(decodedToken)
+	}
 	return
 }
 

cmd/api-resources_test.go

@@ -34,7 +34,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		{
 			values: url.Values{
 				"prefix":             []string{"photos/"},
-				"continuation-token": []string{"token"},
+				"continuation-token": []string{"dG9rZW4="},
 				"start-after":        []string{"start-after"},
 				"delimiter":          []string{SlashSeparator},
 				"fetch-owner":        []string{"true"},
@@ -53,7 +53,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		{
 			values: url.Values{
 				"prefix":             []string{"photos/"},
-				"continuation-token": []string{"token"},
+				"continuation-token": []string{"dG9rZW4="},
 				"start-after":        []string{"start-after"},
 				"delimiter":          []string{SlashSeparator},
 				"fetch-owner":        []string{"true"},
@@ -64,7 +64,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 			startAfter:   "start-after",
 			delimiter:    SlashSeparator,
 			fetchOwner:   true,
-			maxKeys:      1000,
+			maxKeys:      maxObjectList,
 			encodingType: "gzip",
 			errCode:      ErrNone,
 		},
@@ -150,7 +150,7 @@ func TestListObjectsV1Resources(t *testing.T) {
 			prefix:       "photos/",
 			marker:       "test",
 			delimiter:    SlashSeparator,
-			maxKeys:      1000,
+			maxKeys:      maxObjectList,
 			encodingType: "gzip",
 		},
 	}

cmd/api-response.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/xml"
 	"net/http"
 	"net/url"
@@ -33,9 +34,9 @@ import (
 
 const (
 	timeFormatAMZLong = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse.
-	maxUploadsList    = 1000                       // Limit number of uploads in a listUploadsResponse.
-	maxPartsList      = 1000                       // Limit number of parts in a listPartsResponse.
+	maxObjectList     = 10000                      // Limit number of objects in a listObjectsResponse.
+	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
+	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
 )
 
 // LocationResponse - format for location response.
@@ -238,6 +239,37 @@ type ObjectVersion struct {
 	IsLatest  bool
 }
 
+// StringMap is a map[string]string.
+type StringMap map[string]string
+
+// MarshalXML - StringMap marshals into XML.
+func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+
+	tokens := []xml.Token{start}
+
+	for key, value := range s {
+		t := xml.StartElement{}
+		t.Name = xml.Name{
+			Space: "",
+			Local: key,
+		}
+		tokens = append(tokens, t, xml.CharData(value), xml.EndElement{Name: t.Name})
+	}
+
+	tokens = append(tokens, xml.EndElement{
+		Name: start.Name,
+	})
+
+	for _, t := range tokens {
+		if err := e.EncodeToken(t); err != nil {
+			return err
+		}
+	}
+
+	// flush to ensure tokens are written
+	return e.Flush()
+}
+
 // Object container for object metadata
 type Object struct {
 	Key          string
@@ -250,6 +282,9 @@ type Object struct {
 
 	// The class of storage used to store the object.
 	StorageClass string
+
+	// UserMetadata user-defined metadata
+	UserMetadata StringMap `xml:"UserMetadata,omitempty"`
 }
 
 // CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -465,7 +500,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
 	var contents []Object
 	var commonPrefixes []CommonPrefix
 	var owner = Owner{}
@@ -488,6 +523,17 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		content.Size = object.Size
 		content.StorageClass = object.StorageClass
 		content.Owner = owner
+		if metadata {
+			content.UserMetadata = make(StringMap)
+			for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
+				if HasPrefix(k, ReservedMetadataPrefix) {
+					// Do not need to send any internal metadata
+					// values to client.
+					continue
+				}
+				content.UserMetadata[k] = v
+			}
+		}
 		contents = append(contents, content)
 	}
 	data.Name = bucket
@@ -498,8 +544,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	data.Delimiter = s3EncodeName(delimiter, encodingType)
 	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.MaxKeys = maxKeys
-	data.ContinuationToken = token
-	data.NextContinuationToken = nextToken
+	data.ContinuationToken = base64.StdEncoding.EncodeToString([]byte(token))
+	data.NextContinuationToken = base64.StdEncoding.EncodeToString([]byte(nextToken))
 	data.IsTruncated = isTruncated
 	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
@@ -675,7 +721,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
-		if browser && globalIsBrowserEnabled {
+		if browser && globalBrowserEnabled {
 			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return
@@ -693,6 +739,11 @@ func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
 	writeResponse(w, err.HTTPStatusCode, nil, mimeNone)
 }
 
+func writeErrorResponseString(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
+	// Generate string error response.
+	writeResponse(w, err.HTTPStatusCode, []byte(err.Description), mimeNone)
+}
+
 // writeErrorResponseJSON - writes error response in JSON format;
 // useful for admin APIs.
 func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
@@ -702,6 +753,17 @@ func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIE
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
 
+// writeVersionMismatchResponse - writes custom error responses for version mismatches.
+func writeVersionMismatchResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL, isJSON bool) {
+	if isJSON {
+		// Generate error response.
+		errorResponse := getAPIErrorResponse(ctx, err, reqURL.String(), w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+		writeResponse(w, err.HTTPStatusCode, encodeResponseJSON(errorResponse), mimeJSON)
+	} else {
+		writeResponse(w, err.HTTPStatusCode, []byte(err.Description), mimeNone)
+	}
+}
+
 // writeCustomErrorResponseJSON - similar to writeErrorResponseJSON,
 // but accepts the error message directly (this allows messages to be
 // dynamically generated.)
@@ -735,7 +797,7 @@ func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err
 	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
-		if browser && globalIsBrowserEnabled {
+		if browser && globalBrowserEnabled {
 			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return

cmd/api-router.go

@@ -23,6 +23,37 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 )
 
+func newHTTPServerFn() *xhttp.Server {
+	globalObjLayerMutex.Lock()
+	defer globalObjLayerMutex.Unlock()
+	return globalHTTPServer
+}
+
+func newObjectLayerWithoutSafeModeFn() ObjectLayer {
+	globalObjLayerMutex.Lock()
+	defer globalObjLayerMutex.Unlock()
+	return globalObjectAPI
+}
+
+func newObjectLayerFn() ObjectLayer {
+	globalObjLayerMutex.Lock()
+	defer globalObjLayerMutex.Unlock()
+	if globalSafeMode {
+		return nil
+	}
+	return globalObjectAPI
+}
+
+func newCachedObjectLayerFn() CacheObjectLayer {
+	globalObjLayerMutex.Lock()
+	defer globalObjLayerMutex.Unlock()
+
+	if globalSafeMode {
+		return nil
+	}
+	return globalCacheObjectAPI
+}
+
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
@@ -38,7 +69,7 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
-		CacheAPI:  newCacheObjectsFn,
+		CacheAPI:  newCachedObjectLayerFn,
 		EncryptionEnabled: func() bool {
 			return encryptionEnabled
 		},
@@ -59,108 +90,126 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	for _, bucket := range routers {
 		// Object operations
 		// HeadObject
-		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
+		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler)))
 		// CopyObjectPart
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("listobjectparts", httpTraceAll(api.ListObjectPartsHandler))).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
-		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("completemutipartupload", httpTraceAll(api.CompleteMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
-		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("newmultipartupload", httpTraceAll(api.NewMultipartUploadHandler))).Queries("uploads", "")
 		// AbortMultipartUpload
-		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler))).Queries("acl", "")
 		// GetObjectTagging - this is a dummy call.
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler))).Queries("tagging", "")
 		// SelectObjectContent
-		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2")
+		// GetObjectRetention
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectretention", httpTraceHdrs(api.GetObjectRetentionHandler))).Queries("retention", "")
 		// GetObject
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler)))
 		// CopyObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler)))
+		// PutObjectRetention
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectretention", httpTraceHdrs(api.PutObjectRetentionHandler))).Queries("retention", "")
 		// PutObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler)))
 		// DeleteObject
-		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler)))
+		// PutObjectLegalHold
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectlegalhold", httpTraceHdrs(api.PutObjectLegalHoldHandler))).Queries("legal-hold", "")
+		// GetObjectLegalHold
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectlegalhold", httpTraceHdrs(api.GetObjectLegalHoldHandler))).Queries("legal-hold", "")
 
 		/// Bucket operations
 		// GetBucketLocation
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlocation", httpTraceAll(api.GetBucketLocationHandler))).Queries("location", "")
 		// GetBucketPolicy
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler))).Queries("policy", "")
 		// GetBucketLifecycle
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler))).Queries("lifecycle", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler))).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler))).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
-		// GetBucketVersioningHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketwebsite", httpTraceAll(api.GetBucketWebsiteHandler))).Queries("website", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketaccelerate", httpTraceAll(api.GetBucketAccelerateHandler))).Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketrequestpayment", httpTraceAll(api.GetBucketRequestPaymentHandler))).Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlogging", httpTraceAll(api.GetBucketLoggingHandler))).Queries("logging", "")
 		// GetBucketLifecycleHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler))).Queries("lifecycle", "")
 		// GetBucketReplicationHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketreplication", httpTraceAll(api.GetBucketReplicationHandler))).Queries("replication", "")
 		// GetBucketTaggingHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler))).Queries("tagging", "")
 		//DeleteBucketWebsiteHandler
-		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketwebsite", httpTraceAll(api.DeleteBucketWebsiteHandler))).Queries("website", "")
 		// DeleteBucketTaggingHandler
-		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebuckettagging", httpTraceAll(api.DeleteBucketTaggingHandler))).Queries("tagging", "")
 
+		// GetBucketObjectLockConfig
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketobjectlockconfiguration", httpTraceAll(api.GetBucketObjectLockConfigHandler))).Queries("object-lock", "")
+		// GetBucketVersioning
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketversioning", httpTraceAll(api.GetBucketVersioningHandler))).Queries("versioning", "")
 		// GetBucketNotification
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler))).Queries("notification", "")
 		// ListenBucketNotification
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listenbucketnotification", httpTraceAll(api.ListenBucketNotificationHandler))).Queries("events", "{events:.*}")
 		// ListMultipartUploads
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listmultipartuploads", httpTraceAll(api.ListMultipartUploadsHandler))).Queries("uploads", "")
+		// ListObjectsV2M
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listobjectsv2M", httpTraceAll(api.ListObjectsV2MHandler))).Queries("list-type", "2", "metadata", "true")
 		// ListObjectsV2
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler))).Queries("list-type", "2")
 		// ListBucketVersions
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListBucketObjectVersionsHandler)).Queries("versions", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listbucketversions", httpTraceAll(api.ListBucketObjectVersionsHandler))).Queries("versions", "")
 		// ListObjectsV1 (Legacy)
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler)))
 		// PutBucketLifecycle
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketLifecycleHandler)).Queries("lifecycle", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler))).Queries("lifecycle", "")
 		// PutBucketPolicy
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler))).Queries("policy", "")
 
+		// PutBucketObjectLockConfig
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketobjectlockconfig", httpTraceAll(api.PutBucketObjectLockConfigHandler))).Queries("object-lock", "")
+		// PutBucketVersioning
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketversioning", httpTraceAll(api.PutBucketVersioningHandler))).Queries("versioning", "")
 		// PutBucketNotification
-		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketnotification", httpTraceAll(api.PutBucketNotificationHandler))).Queries("notification", "")
 		// PutBucket
-		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketHandler))
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucket", httpTraceAll(api.PutBucketHandler)))
 		// HeadBucket
-		bucket.Methods(http.MethodHead).HandlerFunc(httpTraceAll(api.HeadBucketHandler))
+		bucket.Methods(http.MethodHead).HandlerFunc(collectAPIStats("headbucket", httpTraceAll(api.HeadBucketHandler)))
 		// PostPolicy
-		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
+		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(collectAPIStats("postpolicybucket", httpTraceHdrs(api.PostPolicyBucketHandler)))
 		// DeleteMultipleObjects
-		bucket.Methods(http.MethodPost).HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
+		bucket.Methods(http.MethodPost).HandlerFunc(collectAPIStats("deletemultipleobjects", httpTraceAll(api.DeleteMultipleObjectsHandler))).Queries("delete", "")
 		// DeleteBucketPolicy
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler))).Queries("policy", "")
 		// DeleteBucketLifecycle
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketLifecycleHandler)).Queries("lifecycle", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler))).Queries("lifecycle", "")
 		// DeleteBucket
-		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler)))
 	}
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler)))
+
+	// If none of the routes match add default error handler routes
+	apiRouter.NotFoundHandler = http.HandlerFunc(collectAPIStats("notfound", httpTraceAll(errorResponseHandler)))
+	apiRouter.MethodNotAllowedHandler = http.HandlerFunc(collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler)))
 
-	// If none of the routes match.
-	apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
 }

cmd/auth-handler.go

@@ -120,20 +120,16 @@ func getRequestAuthType(r *http.Request) authType {
 
 // checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
 // It does not accept presigned or JWT or anonymous requests.
-func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region string) APIErrorCode {
+func checkAdminRequestAuthType(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
+	var cred auth.Credentials
+	var owner bool
 	s3Err := ErrAccessDenied
 	if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
 		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
 		// We only support admin credentials to access admin APIs.
-
-		var owner bool
-		_, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
+		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {
-			return s3Err
-		}
-
-		if !owner {
-			return ErrAccessDenied
+			return cred, s3Err
 		}
 
 		// we only support V4 (no presign) with auth body
@@ -142,9 +138,27 @@ func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region stri
 	if s3Err != ErrNone {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
 		ctx := logger.SetReqInfo(ctx, reqInfo)
-		logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description))
+		logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description), logger.Application)
 	}
-	return s3Err
+
+	var claims map[string]interface{}
+	claims, s3Err = checkClaimsFromToken(r, cred)
+	if s3Err != ErrNone {
+		return cred, s3Err
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
+		Action:          iampolicy.Action(action),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		IsOwner:         owner,
+		Claims:          claims,
+	}) {
+		// Request is allowed return the appropriate access key.
+		return cred, ErrNone
+	}
+
+	return cred, ErrAccessDenied
 }
 
 // Fetch the security token set by the client.
@@ -179,7 +193,7 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 		// hijacking the policies. We need to make sure that this is
 		// based an admin credential such that token cannot be decoded
 		// on the client side and is treated like an opaque value.
-		return []byte(globalServerConfig.GetCredential().SecretKey), nil
+		return []byte(globalActiveCred.SecretKey), nil
 	}
 	p := &jwtgo.Parser{
 		ValidMethods: []string{
@@ -203,10 +217,16 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 	}
 
 	if globalPolicyOPA == nil {
+		// If OPA is not set and if ldap claim key is set,
+		// allow the claim.
+		if _, ok := claims[ldapUser]; ok {
+			return claims, nil
+		}
+
 		// If OPA is not set, session token should
 		// have a policy and its mandatory, reject
 		// requests without policy claim.
-		p, pok := claims[iampolicy.PolicyName]
+		p, pok := claims[iamPolicyName()]
 		if !pok {
 			return nil, errAuthentication
 		}
@@ -229,7 +249,7 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
 			// something is malforming the request sent by client.
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(context.Background(), err, logger.Application)
 			return nil, errAuthentication
 		}
 		claims[iampolicy.SessionPolicyName] = string(spBytes)
@@ -280,7 +300,7 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
-		region := globalServerConfig.GetRegion()
+		region := globalServerRegion
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
@@ -306,7 +326,7 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// To extract region from XML in request body, get copy of request body.
 		payload, err := ioutil.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
 		if err != nil {
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, err, logger.Application)
 			return accessKey, owner, ErrMalformedXML
 		}
 
@@ -328,7 +348,7 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 			AccountName:     cred.AccessKey,
 			Action:          action,
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, locationConstraint, ""),
+			ConditionValues: getConditionValues(r, locationConstraint, "", nil),
 			IsOwner:         false,
 			ObjectName:      objectName,
 		}) {
@@ -337,12 +357,11 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		}
 		return accessKey, owner, ErrAccessDenied
 	}
-
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
 		Action:          iampolicy.Action(action),
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 		ObjectName:      objectName,
 		IsOwner:         owner,
 		Claims:          claims,
@@ -467,10 +486,10 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
 }
 
-// isPutAllowed - check if PUT operation is allowed on the resource, this
+// isPutActionAllowed - check if PUT operation is allowed on the resource, this
 // call verifies bucket policies and IAM policies, supports multi user
 // checks etc.
-func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request) (s3Err APIErrorCode) {
+func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	switch atype {
@@ -479,7 +498,7 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
 	case authTypeSignedV2, authTypePresignedV2:
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
-		region := globalServerConfig.GetRegion()
+		region := globalServerRegion
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
@@ -496,7 +515,7 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
 			AccountName:     cred.AccessKey,
 			Action:          policy.PutObjectAction,
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, "", ""),
+			ConditionValues: getConditionValues(r, "", "", nil),
 			IsOwner:         false,
 			ObjectName:      objectName,
 		}) {
@@ -507,9 +526,9 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
 
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
-		Action:          policy.PutObjectAction,
+		Action:          action,
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 		ObjectName:      objectName,
 		IsOwner:         owner,
 		Claims:          claims,

cmd/auth-handler_test.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/pkg/auth"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 )
 
 // Test get request auth type.
@@ -273,7 +274,7 @@ func mustNewRequest(method string, urlStr string, contentLength int64, body io.R
 // is signed with AWS Signature V4, fails if not able to do so.
 func mustNewSignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
-	cred := globalServerConfig.GetCredential()
+	cred := globalActiveCred
 	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
 		t.Fatalf("Unable to inititalized new signed http request %s", err)
 	}
@@ -284,7 +285,7 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod
 // is signed with AWS Signature V2, fails if not able to do so.
 func mustNewSignedV2Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
-	cred := globalServerConfig.GetCredential()
+	cred := globalActiveCred
 	if err := signRequestV2(req, cred.AccessKey, cred.SecretKey); err != nil {
 		t.Fatalf("Unable to inititalized new signed http request %s", err)
 	}
@@ -295,7 +296,7 @@ func mustNewSignedV2Request(method string, urlStr string, contentLength int64, b
 // is presigned with AWS Signature V2, fails if not able to do so.
 func mustNewPresignedV2Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
-	cred := globalServerConfig.GetCredential()
+	cred := globalActiveCred
 	if err := preSignV2(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
 		t.Fatalf("Unable to inititalized new signed http request %s", err)
 	}
@@ -306,7 +307,7 @@ func mustNewPresignedV2Request(method string, urlStr string, contentLength int64
 // is presigned with AWS Signature V4, fails if not able to do so.
 func mustNewPresignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
-	cred := globalServerConfig.GetCredential()
+	cred := globalActiveCred
 	if err := preSignV4(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
 		t.Fatalf("Unable to inititalized new signed http request %s", err)
 	}
@@ -316,7 +317,7 @@ func mustNewPresignedRequest(method string, urlStr string, contentLength int64,
 func mustNewSignedShortMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	req.Header.Set("Content-Md5", "invalid-digest")
-	cred := globalServerConfig.GetCredential()
+	cred := globalActiveCred
 	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
 		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
@@ -326,17 +327,18 @@ func mustNewSignedShortMD5Request(method string, urlStr string, contentLength in
 func mustNewSignedEmptyMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	req.Header.Set("Content-Md5", "")
-	cred := globalServerConfig.GetCredential()
+	cred := globalActiveCred
 	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
 		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
 	return req
 }
 
-func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
+func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64,
+	body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	req.Header.Set("Content-Md5", "YWFhYWFhYWFhYWFhYWFhCg==")
-	cred := globalServerConfig.GetCredential()
+	cred := globalActiveCred
 	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
 		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
@@ -359,7 +361,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 		t.Fatalf("unable create credential, %s", err)
 	}
 
-	globalServerConfig.SetCredential(creds)
+	globalActiveCred = creds
 
 	// List of test cases for validating http request authentication.
 	testCases := []struct {
@@ -381,7 +383,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 	ctx := context.Background()
 	// Validates all testcases.
 	for i, testCase := range testCases {
-		s3Error := isReqAuthenticated(ctx, testCase.req, globalServerConfig.GetRegion(), serviceS3)
+		s3Error := isReqAuthenticated(ctx, testCase.req, globalServerRegion, serviceS3)
 		if s3Error != testCase.s3Error {
 			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
 				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
@@ -405,7 +407,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		t.Fatalf("unable create credential, %s", err)
 	}
 
-	globalServerConfig.SetCredential(creds)
+	globalActiveCred = creds
 	testCases := []struct {
 		Request *http.Request
 		ErrCode APIErrorCode
@@ -418,7 +420,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 	}
 	ctx := context.Background()
 	for i, testCase := range testCases {
-		if s3Error := checkAdminRequestAuthType(ctx, testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuthType(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}

cmd/background-heal-ops.go

@@ -61,11 +61,11 @@ func (h *healRoutine) run() {
 			if !ok {
 				break
 			}
-			if globalHTTPServer != nil {
+			if httpServer := newHTTPServerFn(); httpServer != nil {
 				// Wait at max 10 minute for an inprogress request before proceeding to heal
 				waitCount := 600
 				// Any requests in progress, delay the heal.
-				for (globalHTTPServer.GetRequestCount() >= int32(globalXLSetCount*globalXLSetDriveCount)) &&
+				for (httpServer.GetRequestCount() >= int32(globalEndpoints.Nodes())) &&
 					waitCount > 0 {
 					waitCount--
 					time.Sleep(1 * time.Second)
@@ -83,7 +83,9 @@ func (h *healRoutine) run() {
 			case bucket != "" && object != "":
 				res, err = bgHealObject(ctx, bucket, object, task.opts)
 			}
-			task.responseCh <- healResult{result: res, err: err}
+			if task.responseCh != nil {
+				task.responseCh <- healResult{result: res, err: err}
+			}
 		case <-h.doneCh:
 			return
 		case <-GlobalServiceDoneCh:
@@ -100,18 +102,40 @@ func initHealRoutine() *healRoutine {
 
 }
 
-func initBackgroundHealing() {
-	healBg := initHealRoutine()
-	go healBg.run()
+func startBackgroundHealing() {
+	ctx := context.Background()
 
-	globalBackgroundHealing = healBg
+	var objAPI ObjectLayer
+	for {
+		objAPI = newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		break
+	}
+
+	// Run the background healer
+	globalBackgroundHealRoutine = initHealRoutine()
+	go globalBackgroundHealRoutine.run()
+
+	// Launch the background healer sequence to track
+	// background healing operations
+	info := objAPI.StorageInfo(ctx)
+	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
+	nh := newBgHealSequence(numDisks)
+	globalBackgroundHealState.LaunchNewHealSequence(nh)
+}
+
+func initBackgroundHealing() {
+	go startBackgroundHealing()
 }
 
 // bgHealDiskFormat - heals format.json, return value indicates if a
 // failure error occurred.
 func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return madmin.HealResultItem{}, errServerNotInitialized
 	}
@@ -141,7 +165,7 @@ func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealRes
 // bghealBucket - traverses and heals given bucket
 func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return madmin.HealResultItem{}, errServerNotInitialized
 	}
@@ -152,7 +176,7 @@ func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (mad
 // bgHealObject - heal the given object and record result
 func bgHealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
+	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return madmin.HealResultItem{}, errServerNotInitialized
 	}

cmd/background-newdisks-heal-ops.go

@@ -0,0 +1,127 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"time"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+const defaultMonitorNewDiskInterval = time.Minute * 10
+
+func initLocalDisksAutoHeal() {
+	go monitorLocalDisksAndHeal()
+}
+
+// monitorLocalDisksAndHeal - ensures that detected new disks are healed
+//  1. Only the concerned erasure set will be listed and healed
+//  2. Only the node hosting the disk is responsible to perform the heal
+func monitorLocalDisksAndHeal() {
+	// Wait until the object layer is ready
+	var objAPI ObjectLayer
+	for {
+		objAPI = newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		break
+	}
+
+	z, ok := objAPI.(*xlZones)
+	if !ok {
+		return
+	}
+
+	ctx := context.Background()
+
+	var bgSeq *healSequence
+	var found bool
+
+	for {
+		bgSeq, found = globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
+		if found {
+			break
+		}
+		time.Sleep(time.Second)
+	}
+
+	// Perform automatic disk healing when a new one is inserted
+	for {
+		time.Sleep(defaultMonitorNewDiskInterval)
+
+		localDisksInZoneHeal := make([]Endpoints, len(z.zones))
+		for i, ep := range globalEndpoints {
+			localDisksToHeal := Endpoints{}
+			for _, endpoint := range ep.Endpoints {
+				if !endpoint.IsLocal {
+					continue
+				}
+				// Try to connect to the current endpoint
+				// and reformat if the current disk is not formatted
+				_, _, err := connectEndpoint(endpoint)
+				if err == errUnformattedDisk {
+					localDisksToHeal = append(localDisksToHeal, endpoint)
+				}
+			}
+			if len(localDisksToHeal) == 0 {
+				continue
+			}
+			localDisksInZoneHeal[i] = localDisksToHeal
+		}
+
+		// Reformat disks
+		bgSeq.sourceCh <- SlashSeparator
+		// Ensure that reformatting disks is finished
+		bgSeq.sourceCh <- nopHeal
+
+		var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
+		// Compute the list of erasure set to heal
+		for i, localDisksToHeal := range localDisksInZoneHeal {
+			var erasureSetToHeal []int
+			for _, endpoint := range localDisksToHeal {
+				// Load the new format of this passed endpoint
+				_, format, err := connectEndpoint(endpoint)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					continue
+				}
+				// Calculate the set index where the current endpoint belongs
+				setIndex, _, err := findDiskIndex(z.zones[i].format, format)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					continue
+				}
+
+				erasureSetToHeal = append(erasureSetToHeal, setIndex)
+			}
+			erasureSetInZoneToHeal[i] = erasureSetToHeal
+		}
+
+		// Heal all erasure sets that need
+		for i, erasureSetToHeal := range erasureSetInZoneToHeal {
+			for _, setIndex := range erasureSetToHeal {
+				err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex])
+				if err != nil {
+					logger.LogIf(ctx, err)
+				}
+			}
+		}
+	}
+}

cmd/bitrot-streaming.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/hex"
+	"fmt"
 	"hash"
 	"io"
 
@@ -131,7 +132,8 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err = HashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
+		err = fmt.Errorf("hashes do not match expected %s, got %s",
+			hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))
 		logger.LogIf(context.Background(), err)
 		return 0, err
 	}

cmd/bitrot.go

@@ -155,3 +155,11 @@ func bitrotWriterSum(w io.Writer) []byte {
 	}
 	return nil
 }
+
+// Returns the size of the file with bitrot protection
+func bitrotShardFileSize(size int64, shardSize int64, algo BitrotAlgorithm) int64 {
+	if algo != HighwayHash256S {
+		return size
+	}
+	return ceilFrac(size, shardSize)*int64(algo.New().Size()) + size
+}

cmd/bootstrap-peer-server.go

@@ -0,0 +1,257 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"runtime"
+	"sync/atomic"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio-go/pkg/set"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/rest"
+)
+
+const (
+	bootstrapRESTVersion       = "v1"
+	bootstrapRESTVersionPrefix = SlashSeparator + bootstrapRESTVersion
+	bootstrapRESTPrefix        = minioReservedBucketPath + "/bootstrap"
+	bootstrapRESTPath          = bootstrapRESTPrefix + bootstrapRESTVersionPrefix
+)
+
+const (
+	bootstrapRESTMethodVerify = "/verify"
+)
+
+// To abstract a node over network.
+type bootstrapRESTServer struct{}
+
+// ServerSystemConfig - captures information about server configuration.
+type ServerSystemConfig struct {
+	MinioPlatform  string
+	MinioRuntime   string
+	MinioEndpoints EndpointZones
+}
+
+// Diff - returns error on first difference found in two configs.
+func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
+	if s1.MinioPlatform != s2.MinioPlatform {
+		return fmt.Errorf("Expected platform '%s', found to be running '%s'",
+			s1.MinioPlatform, s2.MinioPlatform)
+	}
+	if s1.MinioEndpoints.Nodes() != s2.MinioEndpoints.Nodes() {
+		return fmt.Errorf("Expected number of endpoints %d, seen %d", s1.MinioEndpoints.Nodes(),
+			s2.MinioEndpoints.Nodes())
+	}
+
+	for i, ep := range s1.MinioEndpoints {
+		if ep.SetCount != s2.MinioEndpoints[i].SetCount {
+			return fmt.Errorf("Expected set count %d, seen %d", ep.SetCount,
+				s2.MinioEndpoints[i].SetCount)
+		}
+		if ep.DrivesPerSet != s2.MinioEndpoints[i].DrivesPerSet {
+			return fmt.Errorf("Expected drives pet set %d, seen %d", ep.DrivesPerSet,
+				s2.MinioEndpoints[i].DrivesPerSet)
+		}
+		for j, endpoint := range ep.Endpoints {
+			if endpoint.String() != s2.MinioEndpoints[i].Endpoints[j].String() {
+				return fmt.Errorf("Expected endpoint %s, seen %s", endpoint,
+					s2.MinioEndpoints[i].Endpoints[j])
+			}
+		}
+
+	}
+	return nil
+}
+
+func getServerSystemCfg() ServerSystemConfig {
+	return ServerSystemConfig{
+		MinioPlatform:  fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
+		MinioEndpoints: globalEndpoints,
+	}
+}
+
+func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "VerifyHandler")
+	cfg := getServerSystemCfg()
+	logger.LogIf(ctx, json.NewEncoder(w).Encode(&cfg))
+	w.(http.Flusher).Flush()
+}
+
+// registerBootstrapRESTHandlers - register bootstrap rest router.
+func registerBootstrapRESTHandlers(router *mux.Router) {
+	server := &bootstrapRESTServer{}
+	subrouter := router.PathPrefix(bootstrapRESTPrefix).Subrouter()
+
+	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodVerify).HandlerFunc(
+		httpTraceHdrs(server.VerifyHandler))
+}
+
+// client to talk to bootstrap Nodes.
+type bootstrapRESTClient struct {
+	endpoint   Endpoint
+	restClient *rest.Client
+	connected  int32
+}
+
+// Reconnect to a bootstrap rest server.k
+func (client *bootstrapRESTClient) reConnect() {
+	atomic.StoreInt32(&client.connected, 1)
+}
+
+// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
+// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
+// after verifying format.json
+func (client *bootstrapRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+	return client.callWithContext(context.Background(), method, values, body, length)
+}
+
+// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
+// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
+// after verifying format.json
+func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+	if !client.IsOnline() {
+		client.reConnect()
+	}
+
+	if values == nil {
+		values = make(url.Values)
+	}
+
+	respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length)
+	if err == nil {
+		return respBody, nil
+	}
+
+	if isNetworkError(err) {
+		atomic.StoreInt32(&client.connected, 0)
+	}
+
+	return nil, err
+}
+
+// Stringer provides a canonicalized representation of node.
+func (client *bootstrapRESTClient) String() string {
+	return client.endpoint.String()
+}
+
+// IsOnline - returns whether RPC client failed to connect or not.
+func (client *bootstrapRESTClient) IsOnline() bool {
+	return atomic.LoadInt32(&client.connected) == 1
+}
+
+// Close - marks the client as closed.
+func (client *bootstrapRESTClient) Close() error {
+	atomic.StoreInt32(&client.connected, 0)
+	client.restClient.Close()
+	return nil
+}
+
+// Verify - fetches system server config.
+func (client *bootstrapRESTClient) Verify(srcCfg ServerSystemConfig) (err error) {
+	if newObjectLayerFn() != nil {
+		return nil
+	}
+	respBody, err := client.call(bootstrapRESTMethodVerify, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer xhttp.DrainBody(respBody)
+	recvCfg := ServerSystemConfig{}
+	if err = json.NewDecoder(respBody).Decode(&recvCfg); err != nil {
+		return err
+	}
+	return srcCfg.Diff(recvCfg)
+}
+
+func verifyServerSystemConfig(endpointZones EndpointZones) error {
+	srcCfg := getServerSystemCfg()
+	clnts := newBootstrapRESTClients(endpointZones)
+	var onlineServers int
+	for onlineServers < len(clnts)/2 {
+		for _, clnt := range clnts {
+			if err := clnt.Verify(srcCfg); err != nil {
+				if isNetworkError(err) {
+					continue
+				}
+				return fmt.Errorf("%s as has incorrect configuration: %w", clnt.String(), err)
+			}
+			onlineServers++
+		}
+		// Sleep for a while - so that we don't go into
+		// 100% CPU when half the endpoints are offline.
+		time.Sleep(500 * time.Millisecond)
+	}
+	return nil
+}
+
+func newBootstrapRESTClients(endpointZones EndpointZones) []*bootstrapRESTClient {
+	seenHosts := set.NewStringSet()
+	var clnts []*bootstrapRESTClient
+	for _, ep := range endpointZones {
+		for _, endpoint := range ep.Endpoints {
+			if seenHosts.Contains(endpoint.Host) {
+				continue
+			}
+			seenHosts.Add(endpoint.Host)
+
+			// Only proceed for remote endpoints.
+			if !endpoint.IsLocal {
+				clnt, err := newBootstrapRESTClient(endpoint)
+				if err != nil {
+					continue
+				}
+				clnts = append(clnts, clnt)
+			}
+		}
+	}
+	return clnts
+}
+
+// Returns a new bootstrap client.
+func newBootstrapRESTClient(endpoint Endpoint) (*bootstrapRESTClient, error) {
+	serverURL := &url.URL{
+		Scheme: endpoint.Scheme,
+		Host:   endpoint.Host,
+		Path:   bootstrapRESTPath,
+	}
+
+	var tlsConfig *tls.Config
+	if globalIsSSL {
+		tlsConfig = &tls.Config{
+			ServerName: endpoint.Hostname(),
+			RootCAs:    globalRootCAs,
+		}
+	}
+
+	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout)
+	restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
+	if err != nil {
+		return nil, err
+	}
+
+	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient, connected: 1}, nil
+}

cmd/bucket-handlers-listobjects.go

@@ -33,7 +33,7 @@ import (
 // - delimiter if set should be equal to '/', otherwise the request is rejected.
 // - marker if set should have a common prefix with 'prefix' param, otherwise
 //   the request is rejected.
-func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
+func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
 	// Max keys cannot be negative.
 	if maxKeys < 0 {
 		return ErrInvalidMaxKeys
@@ -82,7 +82,7 @@ func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWrit
 	}
 
 	// Validate the query params before beginning to serve the request.
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxkeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(marker, delimiter, encodingType, maxkeys); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -126,18 +126,18 @@ func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWrit
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
 
-// ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
+// ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata.
 // --------------------------
-// This implementation of the GET operation returns some or all (up to 1000)
-// of the objects in a bucket. You can use the request parameters as selection
+// This implementation of the GET operation returns some or all (up to 10000)
+// of the objects in a bucket. You can use the request parame<ters as selection
 // criteria to return a subset of the objects in a bucket.
 //
 // NOTE: It is recommended that this API to be used for application development.
-// MinIO continues to support ListObjectsV1 for supporting legacy tools.
-func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListObjectsV2")
+// MinIO continues to support ListObjectsV1 and V2 for supporting legacy tools.
+func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListObjectsV2M")
 
-	defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(w, r, "ListObjectsV2M", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -164,7 +164,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -201,8 +201,94 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		}
 	}
 
-	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
-		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes)
+	response := generateListObjectsV2Response(bucket, prefix, token,
+		listObjectsV2Info.NextContinuationToken, startAfter,
+		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
+		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, true)
+
+	// Write success response.
+	writeSuccessResponseXML(w, encodeResponse(response))
+}
+
+// ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
+// --------------------------
+// This implementation of the GET operation returns some or all (up to 10000)
+// of the objects in a bucket. You can use the request parameters as selection
+// criteria to return a subset of the objects in a bucket.
+//
+// NOTE: It is recommended that this API to be used for application development.
+// MinIO continues to support ListObjectsV1 for supporting legacy tools.
+func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListObjectsV2")
+
+	defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	urlValues := r.URL.Query()
+
+	// Extract all the listObjectsV2 query params to their native values.
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
+	if errCode != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Validate the query params before beginning to serve the request.
+	// fetch-owner is not validated since it is a boolean
+	if s3Error := validateListObjectsArgs(token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	listObjectsV2 := objectAPI.ListObjectsV2
+
+	// Inititate a list objects operation based on the input params.
+	// On success would return back ListObjectsInfo object to be
+	// marshaled into S3 compatible XML header.
+	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	for i := range listObjectsV2Info.Objects {
+		var actualSize int64
+		if listObjectsV2Info.Objects[i].IsCompressed() {
+			// Read the decompressed size from the meta.json.
+			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
+			if actualSize < 0 {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
+				return
+			}
+			// Set the info.Size to the actualSize.
+			listObjectsV2Info.Objects[i].Size = actualSize
+		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
+			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
+			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+				return
+			}
+		}
+	}
+
+	response := generateListObjectsV2Response(bucket, prefix, token,
+		listObjectsV2Info.NextContinuationToken, startAfter,
+		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
+		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))
@@ -210,7 +296,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 // ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
 // --------------------------
-// This implementation of the GET operation returns some or all (up to 1000)
+// This implementation of the GET operation returns some or all (up to 10000)
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
@@ -241,7 +327,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	// Validate all the query params before beginning to serve the request.
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}

cmd/bucket-handlers.go

@@ -32,10 +32,10 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio/cmd/config/etcd/dns"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
@@ -44,18 +44,26 @@ import (
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+const (
+	getBucketVersioningResponse       = `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`
+	objectLockConfig                  = "object-lock.xml"
+	bucketObjectLockEnabledConfigFile = "object-lock-enabled.json"
+	bucketObjectLockEnabledConfig     = `{"x-amz-bucket-object-lock-enabled":true}`
+)
+
 // Check if there are buckets on server without corresponding entry in etcd backend and
 // make entries. Here is the general flow
 // - Range over all the available buckets
 // - Check if a bucket has an entry in etcd backend
 // -- If no, make an entry
-// -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
-// -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
-func initFederatorBackend(objLayer ObjectLayer) {
-	// Get buckets in the backend
-	b, err := objLayer.ListBuckets(context.Background())
-	if err != nil {
-		logger.LogIf(context.Background(), err)
+// -- If yes, check if the entry matches local IP check if we
+//    need to update the entry then proceed to update
+// -- If yes, check if the IP of entry matches local IP.
+//    This means entry is for this instance.
+// -- If IP of the entry doesn't match, this means entry is
+//    for another instance. Log an error to console.
+func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
+	if len(buckets) == 0 {
 		return
 	}
 
@@ -69,23 +77,37 @@ func initFederatorBackend(objLayer ObjectLayer) {
 	bucketSet := set.NewStringSet()
 
 	// Add buckets that are not registered with the DNS
-	g := errgroup.WithNErrs(len(b))
-	for index := range b {
+	g := errgroup.WithNErrs(len(buckets))
+	for index := range buckets {
+		bucketSet.Add(buckets[index].Name)
 		index := index
-		bucketSet.Add(b[index].Name)
 		g.Go(func() error {
-			r, gerr := globalDNSConfig.Get(b[index].Name)
+			r, gerr := globalDNSConfig.Get(buckets[index].Name)
 			if gerr != nil {
 				if gerr == dns.ErrNoEntriesFound {
-					return globalDNSConfig.Put(b[index].Name)
+					return globalDNSConfig.Put(buckets[index].Name)
 				}
 				return gerr
 			}
-			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-				// There is already an entry for this bucket, with all IP addresses different. This indicates a bucket name collision. Log an error and continue.
-				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", b[index].Name, globalDomainIPs.ToSlice())
+			if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+					// No difference in terms of domainIPs and nothing
+					// has changed so we don't change anything on the etcd.
+					return nil
+				}
+				// if domain IPs intersect then it won't be an empty set.
+				// such an intersection means that bucket exists on etcd.
+				// but if we do see a difference with local domain IPs with
+				// hostSlice from etcd then we should update with newer
+				// domainIPs, we proceed to do that here.
+				return globalDNSConfig.Put(buckets[index].Name)
 			}
-			return nil
+			// No IPs seem to intersect, this means that bucket exists but has
+			// different IP addresses perhaps from a different deployment.
+			// bucket names are globally unique in federation at a given
+			// path prefix, name collision is not allowed. We simply log
+			// an error and continue.
+			return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", buckets[index].Name, globalDomainIPs.ToSlice())
 		}, index)
 	}
 
@@ -99,7 +121,6 @@ func initFederatorBackend(objLayer ObjectLayer) {
 	// Remove buckets that are in DNS for this server, but aren't local
 	for index := range dnsBuckets {
 		index := index
-
 		g.Go(func() error {
 			// This is a local bucket that exists, so we can continue
 			if bucketSet.Contains(dnsBuckets[index].Key) {
@@ -107,14 +128,16 @@ func initFederatorBackend(objLayer ObjectLayer) {
 			}
 
 			// This is not for our server, so we can continue
-			hostPort := net.JoinHostPort(dnsBuckets[index].Host, fmt.Sprintf("%d", dnsBuckets[index].Port))
+			hostPort := net.JoinHostPort(dnsBuckets[index].Host, string(dnsBuckets[index].Port))
 			if globalDomainIPs.Intersection(set.CreateStringSet(hostPort)).IsEmpty() {
 				return nil
 			}
 
-			// We go to here, so we know the bucket no longer exists, but is registered in DNS to this server
+			// We go to here, so we know the bucket no longer exists,
+			// but is registered in DNS to this server
 			if err := globalDNSConfig.DeleteRecord(dnsBuckets[index]); err != nil {
-				return fmt.Errorf("Failed to remove DNS entry for %s due to %v", dnsBuckets[index].Key, err)
+				return fmt.Errorf("Failed to remove DNS entry for %s due to %w",
+					dnsBuckets[index].Key, err)
 			}
 
 			return nil
@@ -160,7 +183,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 	// Generate response.
 	encodedSuccessResponse := encodeResponse(LocationResponse{})
 	// Get current region.
-	region := globalServerConfig.GetRegion()
+	region := globalServerRegion
 	if region != globalMinioDefaultRegion {
 		encodedSuccessResponse = encodeResponse(LocationResponse{
 			Location: region,
@@ -211,7 +234,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 
 	if keyMarker != "" {
 		// Marker not common with prefix is not implemented.
-		if !hasPrefix(keyMarker, prefix) {
+		if !HasPrefix(keyMarker, prefix) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 			return
 		}
@@ -297,7 +320,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 			AccountName:     accessKey,
 			Action:          iampolicy.ListBucketAction,
 			BucketName:      bucketInfo.Name,
-			ConditionValues: getConditionValues(r, "", accessKey),
+			ConditionValues: getConditionValues(r, "", accessKey, claims),
 			IsOwner:         owner,
 			ObjectName:      "",
 			Claims:          claims,
@@ -354,7 +377,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
-		logger.LogIf(ctx, err)
+		logger.LogIf(ctx, err, logger.Application)
 		writeErrorResponse(ctx, w, toAdminAPIErr(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -362,30 +385,22 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Unmarshal list of keys to be deleted.
 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
-		logger.LogIf(ctx, err)
+		logger.LogIf(ctx, err, logger.Application)
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		// Not required to check whether given objects exist or not, because
-		// DeleteMultipleObject is always successful irrespective of object existence.
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
 	deleteObjectsFn := objectAPI.DeleteObjects
 	if api.CacheAPI() != nil {
 		deleteObjectsFn = api.CacheAPI().DeleteObjects
 	}
 
-	type delObj struct {
-		origIndex int
-		name      string
+	var objectsToDelete = map[string]int{}
+	getObjectInfoFn := objectAPI.GetObjectInfo
+	if api.CacheAPI() != nil {
+		getObjectInfoFn = api.CacheAPI().GetObjectInfo
 	}
 
-	var objectsToDelete []delObj
 	var dErrs = make([]APIErrorCode, len(deleteObjects.Objects))
 
 	for index, object := range deleteObjects.Objects {
@@ -396,26 +411,37 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			}
 			continue
 		}
-
-		objectsToDelete = append(objectsToDelete, delObj{index, object.ObjectName})
+		govBypassPerms := checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName)
+		if _, err := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn, govBypassPerms); err != ErrNone {
+			dErrs[index] = err
+			continue
+		}
+		// Avoid duplicate objects, we use map to filter them out.
+		if _, ok := objectsToDelete[object.ObjectName]; !ok {
+			objectsToDelete[object.ObjectName] = index
+		}
 	}
 
-	toNames := func(input []delObj) (output []string) {
+	toNames := func(input map[string]int) (output []string) {
 		output = make([]string, len(input))
-		for i := range input {
-			output[i] = input[i].name
+		idx := 0
+		for name := range input {
+			output[idx] = name
+			idx++
 		}
 		return
 	}
 
-	errs, err := deleteObjectsFn(ctx, bucket, toNames(objectsToDelete))
+	deleteList := toNames(objectsToDelete)
+	errs, err := deleteObjectsFn(ctx, bucket, deleteList)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	for i, obj := range objectsToDelete {
-		dErrs[obj.origIndex] = toAPIErrorCode(ctx, errs[i])
+	for i, objName := range deleteList {
+		dIdx := objectsToDelete[objName]
+		dErrs[dIdx] = toAPIErrorCode(ctx, errs[i])
 	}
 
 	// Collect deleted objects and errors if any.
@@ -477,6 +503,16 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
+	objectLockEnabled := false
+	if vs, found := r.Header[http.CanonicalHeaderKey("x-amz-bucket-object-lock-enabled")]; found {
+		v := strings.ToLower(strings.Join(vs, ""))
+		if v != "true" && v != "false" {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
+			return
+		}
+		objectLockEnabled = v == "true"
+	}
+
 	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
@@ -504,6 +540,15 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
+
+				if objectLockEnabled {
+					configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
+					if err = saveConfig(ctx, objectAPI, configFile, []byte(bucketObjectLockEnabledConfig)); err != nil {
+						writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+						return
+					}
+				}
+
 				if err = globalDNSConfig.Put(bucket); err != nil {
 					objectAPI.DeleteBucket(ctx, bucket)
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -532,6 +577,16 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
+	if objectLockEnabled {
+		configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
+		if err = saveConfig(ctx, objectAPI, configFile, []byte(bucketObjectLockEnabledConfig)); err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+		globalBucketObjectLockConfig.Set(bucket, Retention{})
+		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, Retention{})
+	}
+
 	// Make sure to add Location information here only for bucket
 	w.Header().Set(xhttp.Location, path.Clean(r.URL.Path)) // Clean any trailing slashes.
 
@@ -557,8 +612,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
-	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -597,7 +651,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Read multipart data and save in memory and in the disk if needed
 	form, err := reader.ReadForm(maxFormMemory)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		logger.LogIf(ctx, err, logger.Application)
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -608,7 +662,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Extract all form fields
 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		logger.LogIf(ctx, err, logger.Application)
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -657,9 +711,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Handle policy if it is set.
 	if len(policyBytes) > 0 {
+
 		postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
 		if err != nil {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPostPolicyConditionInvalidFormat), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -715,7 +770,11 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, SlashSeparator) { // handle SSE-C and SSE-S3 requests
+		if crypto.IsRequested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
+			if crypto.SSECopy.IsRequested(r.Header) {
+				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
+				return
+			}
 			var reader io.Reader
 			var key []byte
 			if crypto.SSEC.IsRequested(formValues) {
@@ -858,12 +917,194 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 	}
 
-	globalNotificationSys.RemoveNotification(bucket)
-	globalPolicySys.Remove(bucket)
 	globalNotificationSys.DeleteBucket(ctx, bucket)
-	globalLifecycleSys.Remove(bucket)
-	globalNotificationSys.RemoveBucketLifecycle(ctx, bucket)
 
 	// Write success response.
 	writeSuccessNoContent(w)
 }
+
+// PutBucketVersioningHandler - PUT Bucket Versioning.
+// ----------
+// No-op. Available for API compatibility.
+func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketVersioning")
+
+	defer logger.AuditLog(w, r, "PutBucketVersioning", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	getBucketInfo := objectAPI.GetBucketInfo
+	if _, err := getBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketVersioningHandler - GET Bucket Versioning.
+// ----------
+// No-op. Available for API compatibility.
+func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketVersioning")
+
+	defer logger.AuditLog(w, r, "GetBucketVersioning", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	getBucketInfo := objectAPI.GetBucketInfo
+	if _, err := getBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseXML(w, []byte(getBucketVersioningResponse))
+}
+
+// PutBucketObjectLockConfigHandler - PUT Bucket object lock configuration.
+// ----------
+// Places an Object Lock configuration on the specified bucket. The rule
+// specified in the Object Lock configuration will be applied by default
+// to every new object placed in the specified bucket.
+func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketObjectLockConfig")
+
+	defer logger.AuditLog(w, r, "PutBucketObjectLockConfig", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	config, err := parseObjectLockConfig(r.Body)
+	if err != nil {
+		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
+		apiErr.Description = err.Error()
+		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
+		return
+	}
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
+	configData, err := readConfig(ctx, objectAPI, configFile)
+	if err != nil {
+		aerr := toAPIError(ctx, err)
+		if err == errConfigNotFound {
+			aerr = errorCodes.ToAPIErr(ErrMethodNotAllowed)
+		}
+		writeErrorResponse(ctx, w, aerr, r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if string(configData) != bucketObjectLockEnabledConfig {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	data, err := xml.Marshal(config)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	configFile = path.Join(bucketConfigPrefix, bucket, objectLockConfig)
+	if err = saveConfig(ctx, objectAPI, configFile, data); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	if config.Rule != nil {
+		retention := config.ToRetention()
+		globalBucketObjectLockConfig.Set(bucket, retention)
+		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, retention)
+	} else {
+		globalBucketObjectLockConfig.Remove(bucket)
+		globalNotificationSys.RemoveBucketObjectLockConfig(ctx, bucket)
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketObjectLockConfigHandler - GET Bucket object lock configuration.
+// ----------
+// Gets the Object Lock configuration for a bucket. The rule specified in
+// the Object Lock configuration will be applied by default to every new
+// object placed in the specified bucket.
+func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketObjectLockConfig")
+
+	defer logger.AuditLog(w, r, "GetBucketObjectLockConfig", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	// check if user has permissions to perform this operation
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
+	configData, err := readConfig(ctx, objectAPI, configFile)
+	if err != nil {
+		var aerr APIError
+		if err == errConfigNotFound {
+			aerr = errorCodes.ToAPIErr(ErrMethodNotAllowed)
+		} else {
+			aerr = toAPIError(ctx, err)
+		}
+		writeErrorResponse(ctx, w, aerr, r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if string(configData) != bucketObjectLockEnabledConfig {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	configFile = path.Join(bucketConfigPrefix, bucket, objectLockConfig)
+	configData, err = readConfig(ctx, objectAPI, configFile)
+	if err != nil {
+		if err != errConfigNotFound {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if configData, err = xml.Marshal(newObjectLockConfig()); err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
+	// Write success response.
+	writeSuccessResponseXML(w, configData)
+}

cmd/bucket-handlers_test.go

@@ -331,7 +331,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			shouldPass:         false,
 		},
 		// Test case -3.
-		// Setting invalid delimiter, expecting the HTTP response status to be http.StatusNotImplemented.
+		// Delimiter unsupported, but response is empty.
 		{
 			bucket:             bucketName,
 			prefix:             "",
@@ -341,8 +341,8 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			maxUploads:         "0",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusNotImplemented,
-			shouldPass:         false,
+			expectedRespStatus: http.StatusOK,
+			shouldPass:         true,
 		},
 		// Test case - 4.
 		// Setting Invalid prefix and marker combination.

cmd/bucket-lifecycle-handler.go

@@ -80,7 +80,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 	globalNotificationSys.SetBucketLifecycle(ctx, bucket, bucketLifecycle)
 
 	// Success.
-	writeSuccessNoContent(w)
+	writeSuccessResponseHeadersOnly(w)
 }
 
 // GetBucketLifecycleHandler - This HTTP handler returns bucket policy configuration.

cmd/bucket-notification-handlers.go

@@ -17,26 +17,26 @@
 package cmd
 
 import (
-	"bytes"
+	"encoding/json"
 	"encoding/xml"
 	"errors"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
+	"reflect"
+	"time"
 
 	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/event/target"
-	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
 )
 
 const (
 	bucketConfigPrefix       = "buckets"
 	bucketNotificationConfig = "notification.xml"
-	bucketListenerConfig     = "listener.json"
 )
 
 var errNoSuchNotifications = errors.New("The specified bucket does not have bucket notifications")
@@ -51,7 +51,6 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
-	var config *event.Config
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -78,21 +77,55 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 	// Construct path to notification.xml for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)
 
+	var config = event.Config{}
 	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
 		if err != errConfigNotFound {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		config = &event.Config{}
-	} else {
-		if err = xml.NewDecoder(bytes.NewReader(configData)).Decode(&config); err != nil {
+		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
+		config.SetRegion(globalServerRegion)
+		notificationBytes, err := xml.Marshal(config)
+		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
+
+		writeSuccessResponseXML(w, notificationBytes)
+		return
 	}
 
-	config.SetRegion(globalServerConfig.GetRegion())
+	config.SetRegion(globalServerRegion)
+
+	if err = xml.Unmarshal(configData, &config); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
+		arnErr, ok := err.(*event.ErrARNNotFound)
+		if !ok {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+		for i, queue := range config.QueueList {
+			// Remove ARN not found queues, because we previously allowed
+			// adding unexpected entries into the config.
+			//
+			// With newer config disallowing changing / turning off
+			// notification targets without removing ARN in notification
+			// configuration we won't see this problem anymore.
+			if reflect.DeepEqual(queue.ARN, arnErr.ARN) {
+				config.QueueList = append(config.QueueList[:i],
+					config.QueueList[i+1:]...)
+			}
+			// This is a one time activity we shall do this
+			// here and allow stale ARN to be removed. We shall
+			// never reach a stage where we will have stale
+			// notification configs.
+		}
+	}
 
 	// If xml namespace is empty, set a default value before returning.
 	if config.XMLNS == "" {
@@ -146,17 +179,15 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	var config *event.Config
-	config, err = event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalServerConfig.GetRegion(), globalNotificationSys.targetList)
+	lreader := io.LimitReader(r.Body, r.ContentLength)
+	config, err := event.ParseConfig(lreader, globalServerRegion, globalNotificationSys.targetList)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {
 			apiErr = toAPIError(ctx, err)
 		}
-		if _, ok := err.(*event.ErrARNNotFound); !ok {
-			writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
-			return
-		}
+		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
+		return
 	}
 
 	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
@@ -171,8 +202,6 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// ListenBucketNotificationHandler - This HTTP handler sends events to the connected HTTP client.
-// Client should send prefix/suffix object name to match and events to watch as query parameters.
 func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListenBucketNotification")
 
@@ -194,50 +223,47 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
-	if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
 	values := r.URL.Query()
+	values.Set(peerRESTListenBucket, bucketName)
 
 	var prefix string
-	if len(values["prefix"]) > 1 {
+	if len(values[peerRESTListenPrefix]) > 1 {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNamePrefix), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	if len(values["prefix"]) == 1 {
-		if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil {
+	if len(values[peerRESTListenPrefix]) == 1 {
+		if err := event.ValidateFilterRuleValue(values[peerRESTListenPrefix][0]); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
-		prefix = values["prefix"][0]
+		prefix = values[peerRESTListenPrefix][0]
 	}
 
 	var suffix string
-	if len(values["suffix"]) > 1 {
+	if len(values[peerRESTListenSuffix]) > 1 {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNameSuffix), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	if len(values["suffix"]) == 1 {
-		if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil {
+	if len(values[peerRESTListenSuffix]) == 1 {
+		if err := event.ValidateFilterRuleValue(values[peerRESTListenSuffix][0]); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
-		suffix = values["suffix"][0]
+		suffix = values[peerRESTListenSuffix][0]
 	}
 
 	pattern := event.NewPattern(prefix, suffix)
 
-	eventNames := []event.Name{}
-	for _, s := range values["events"] {
+	var eventNames []event.Name
+	for _, s := range values[peerRESTListenEvents] {
 		eventName, err := event.ParseName(s)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -252,49 +278,67 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	host, err := xnet.ParseHost(r.RemoteAddr)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
+	rulesMap := event.NewRulesMap(eventNames, pattern, event.TargetID{ID: mustGetUUID()})
 
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
 
-	target, err := target.NewHTTPClientTarget(*host, w)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
+	// Use buffered channel to take care of burst sends or slow w.Write()
+	listenCh := make(chan interface{}, 4000)
+
+	peers := getRestClients(globalEndpoints)
+
+	globalHTTPListen.Subscribe(listenCh, doneCh, func(evI interface{}) bool {
+		ev, ok := evI.(event.Event)
+		if !ok {
+			return false
+		}
+		if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) {
+			return false
+		}
+		objectName, uerr := url.QueryUnescape(ev.S3.Object.Key)
+		if uerr != nil {
+			objectName = ev.S3.Object.Key
+		}
+		return len(rulesMap.Match(ev.EventName, objectName).ToSlice()) != 0
+	})
+
+	for _, peer := range peers {
+		if peer == nil {
+			continue
+		}
+		peer.Listen(listenCh, doneCh, values)
 	}
 
-	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
 
-	if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
-		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
-	defer globalNotificationSys.RemoveRulesMap(bucketName, rulesMap)
-
-	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
+	enc := json.NewEncoder(w)
+	for {
+		select {
+		case evI := <-listenCh:
+			ev := evI.(event.Event)
+			if len(string(ev.EventName)) > 0 {
+				if err := enc.Encode(struct{ Records []event.Event }{[]event.Event{ev}}); err != nil {
+					return
+				}
+			} else {
+				if _, err := w.Write([]byte(" ")); err != nil {
+					return
+				}
+			}
+			w.(http.Flusher).Flush()
+		case <-keepAliveTicker.C:
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-GlobalServiceDoneCh:
+			return
+		}
 	}
 
-	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
-		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	globalNotificationSys.ListenBucketNotification(ctx, bucketName, eventNames, pattern, target.ID(), *thisAddr)
-
-	<-target.DoneCh
-
-	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
-		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
 }

cmd/common-main.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,25 +17,21 @@
 package cmd
 
 import (
-	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"net"
-	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"time"
 
-	etcd "github.com/coreos/etcd/clientv3"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/logger/target/console"
-	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/dns"
-	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/env"
 )
 
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
@@ -46,12 +42,12 @@ func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
 
 	if strings.HasPrefix(name, "gateway") {
 		if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
-			uiErr := uiErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
+			uiErr := config.ErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
 			logger.Fatal(uiErr, "Unable to start gateway with SSE")
 		}
 	}
 
-	if globalIsCompressionEnabled && !objAPI.IsCompressionSupported() {
+	if globalCompressConfig.Enabled && !objAPI.IsCompressionSupported() {
 		logger.Fatal(errInvalidArgument,
 			"Compression support is requested but '%s' does not support compression", name)
 	}
@@ -65,43 +61,13 @@ func checkUpdate(mode string) {
 			return
 		}
 		if globalInplaceUpdateDisabled {
-			logger.StartupMessage(updateMsg)
+			logStartupMessage(updateMsg)
 		} else {
-			logger.StartupMessage(prepareUpdateMessage("Run `mc admin update`", latestReleaseTime.Sub(currentReleaseTime)))
+			logStartupMessage(prepareUpdateMessage("Run `mc admin update`", latestReleaseTime.Sub(currentReleaseTime)))
 		}
 	}
 }
 
-// Load logger targets based on user's configuration
-func loadLoggers() {
-	loggerUserAgent := getUserAgent(getMinioMode())
-
-	auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
-	if ok {
-		// Enable audit HTTP logging through ENV.
-		logger.AddAuditTarget(http.New(auditEndpoint, loggerUserAgent, NewCustomHTTPTransport()))
-	}
-
-	loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
-	if ok {
-		// Enable HTTP logging through ENV.
-		logger.AddTarget(http.New(loggerEndpoint, loggerUserAgent, NewCustomHTTPTransport()))
-	} else {
-		for _, l := range globalServerConfig.Logger.HTTP {
-			if l.Enabled {
-				// Enable http logging
-				logger.AddTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
-			}
-		}
-	}
-
-	if globalServerConfig.Logger.Console.Enabled {
-		// Enable console logging
-		logger.AddTarget(console.New())
-	}
-
-}
-
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
 	var dir string
 	var dirSet bool
@@ -190,112 +156,27 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	globalCLIContext.StrictS3Compat = ctx.IsSet("compat") || ctx.GlobalIsSet("compat")
 }
 
-// Parses the given compression exclude list `extensions` or `content-types`.
-func parseCompressIncludes(includes []string) ([]string, error) {
-	for _, e := range includes {
-		if len(e) == 0 {
-			return nil, uiErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
-		}
-	}
-	return includes, nil
-}
-
 func handleCommonEnvVars() {
-	compressEnvDelimiter := ","
-	// Start profiler if env is set.
-	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
-		var err error
-		globalProfiler, err = startProfiler(profiler, "")
-		logger.FatalIf(err, "Unable to setup a profiler")
+	var err error
+	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
+	if err != nil {
+		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
 	}
 
-	accessKey := os.Getenv("MINIO_ACCESS_KEY")
-	secretKey := os.Getenv("MINIO_SECRET_KEY")
-	if accessKey != "" && secretKey != "" {
-		cred, err := auth.CreateCredentials(accessKey, secretKey)
-		if err != nil {
-			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
-		}
-		cred.Expiration = timeSentinel
-
-		// credential Envs are set globally.
-		globalIsEnvCreds = true
-		globalActiveCred = cred
-	}
-
-	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
-		browserFlag, err := ParseBoolFlag(browser)
-		if err != nil {
-			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
-		}
-
-		// browser Envs are set globally, this does not represent
-		// if browser is turned off or on.
-		globalIsEnvBrowser = true
-		globalIsBrowserEnabled = bool(browserFlag)
-	}
-
-	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
-	if ok {
-		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
-
-		var etcdSecure bool
-		for _, endpoint := range etcdEndpoints {
-			u, err := xnet.ParseURL(endpoint)
-			if err != nil {
-				logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
-			}
-			// If one of the endpoint is https, we will use https directly.
-			etcdSecure = etcdSecure || u.Scheme == "https"
-		}
-
-		var err error
-		if etcdSecure {
-			// This is only to support client side certificate authentication
-			// https://coreos.com/etcd/docs/latest/op-guide/security.html
-			etcdClientCertFile, ok1 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT")
-			etcdClientCertKey, ok2 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT_KEY")
-			var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
-			if ok1 && ok2 {
-				getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-					cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
-					return &cert, terr
-				}
-			}
-
-			globalEtcdClient, err = etcd.New(etcd.Config{
-				Endpoints:         etcdEndpoints,
-				DialTimeout:       defaultDialTimeout,
-				DialKeepAliveTime: defaultDialKeepAlive,
-				TLS: &tls.Config{
-					RootCAs:              globalRootCAs,
-					GetClientCertificate: getClientCertificate,
-				},
-			})
-		} else {
-			globalEtcdClient, err = etcd.New(etcd.Config{
-				Endpoints:         etcdEndpoints,
-				DialTimeout:       defaultDialTimeout,
-				DialKeepAliveTime: defaultDialKeepAlive,
-			})
-		}
-		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
-	}
-
-	v, ok := os.LookupEnv("MINIO_DOMAIN")
-	if ok {
-		for _, domainName := range strings.Split(v, ",") {
-			if _, ok = dns2.IsDomainName(domainName); !ok {
-				logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
+	domains := env.Get(config.EnvDomain, "")
+	if len(domains) != 0 {
+		for _, domainName := range strings.Split(domains, config.ValueSeparator) {
+			if _, ok := dns2.IsDomainName(domainName); !ok {
+				logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
 					"Invalid MINIO_DOMAIN value in environment variable")
 			}
 			globalDomainNames = append(globalDomainNames, domainName)
 		}
 	}
 
-	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
-	if ok {
-		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
+	publicIPs := env.Get(config.EnvPublicIPs, "")
+	if len(publicIPs) != 0 {
+		minioEndpoints := strings.Split(publicIPs, config.ValueSeparator)
 		var domainIPs = set.NewStringSet()
 		for _, endpoint := range minioEndpoints {
 			if net.ParseIP(endpoint) == nil {
@@ -307,7 +188,6 @@ func handleCommonEnvVars() {
 				for _, addr := range addrs {
 					domainIPs.Add(addr)
 				}
-				continue
 			}
 			domainIPs.Add(endpoint)
 		}
@@ -315,127 +195,51 @@ func handleCommonEnvVars() {
 	} else {
 		// Add found interfaces IP address to global domain IPS,
 		// loopback addresses will be naturally dropped.
-		updateDomainIPs(localIP4)
+		updateDomainIPs(mustGetLocalIP4())
 	}
 
-	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
-		var err error
-		globalDNSConfig, err = dns.NewCoreDNS(globalDomainNames, globalDomainIPs, globalMinioPort, globalEtcdClient)
-		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainNames)
-	}
-
-	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
-		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
-		if err != nil {
-			logger.Fatal(err, "Unable to parse MINIO_CACHE_DRIVES value (%s)", drives)
-		}
-		globalCacheDrives = driveList
-		globalIsDiskCacheEnabled = true
-	}
-
-	if excludes := os.Getenv("MINIO_CACHE_EXCLUDE"); excludes != "" {
-		excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter))
-		if err != nil {
-			logger.Fatal(err, "Unable to parse MINIO_CACHE_EXCLUDE value (`%s`)", excludes)
-		}
-		globalCacheExcludes = excludeList
-	}
-
-	if expiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); expiryStr != "" {
-		expiry, err := strconv.Atoi(expiryStr)
-		if err != nil {
-			logger.Fatal(uiErrInvalidCacheExpiryValue(err), "Unable to parse MINIO_CACHE_EXPIRY value (`%s`)", expiryStr)
-		}
-		globalCacheExpiry = expiry
-	}
-
-	if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" {
-		maxUse, err := strconv.Atoi(maxUseStr)
-		if err != nil {
-			logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr)
-		}
-		// maxUse should be a valid percentage.
-		if maxUse > 0 && maxUse <= 100 {
-			globalCacheMaxUse = maxUse
-		}
-	}
-
-	var err error
-	if cacheEncKey := os.Getenv("MINIO_CACHE_ENCRYPTION_MASTER_KEY"); cacheEncKey != "" {
-		globalCacheKMSKeyID, globalCacheKMS, err = parseKMSMasterKey(cacheEncKey)
-		if err != nil {
-			logger.Fatal(uiErrInvalidCacheEncryptionKey(err), "Invalid cache encryption master key")
-		}
-	}
 	// In place update is true by default if the MINIO_UPDATE is not set
 	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
 	// in-place update is off.
-	globalInplaceUpdateDisabled = strings.EqualFold(os.Getenv("MINIO_UPDATE"), "off")
+	globalInplaceUpdateDisabled = strings.EqualFold(env.Get(config.EnvUpdate, config.EnableOn), config.EnableOff)
 
-	// Validate and store the storage class env variables only for XL/Dist XL setups
-	if globalIsXL {
-		var err error
-
-		// Check for environment variables and parse into storageClass struct
-		if ssc := os.Getenv(standardStorageClassEnv); ssc != "" {
-			globalStandardStorageClass, err = parseStorageClass(ssc)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
-		}
-
-		if rrsc := os.Getenv(reducedRedundancyStorageClassEnv); rrsc != "" {
-			globalRRStorageClass, err = parseStorageClass(rrsc)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
-		}
-
-		// Validation is done after parsing both the storage classes. This is needed because we need one
-		// storage class value to deduce the correct value of the other storage class.
-		if globalRRStorageClass.Scheme != "" {
-			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
-			globalIsStorageClass = true
-		}
-
-		if globalStandardStorageClass.Scheme != "" {
-			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
-			globalIsStorageClass = true
-		}
-	}
-
-	// Get WORM environment variable.
-	if worm := os.Getenv("MINIO_WORM"); worm != "" {
-		wormFlag, err := ParseBoolFlag(worm)
+	if env.IsSet(config.EnvAccessKey) || env.IsSet(config.EnvSecretKey) {
+		cred, err := auth.CreateCredentials(env.Get(config.EnvAccessKey, ""), env.Get(config.EnvSecretKey, ""))
 		if err != nil {
-			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the shell environment")
 		}
-
-		// worm Envs are set globally, this does not represent
-		// if worm is turned off or on.
-		globalIsEnvWORM = true
-		globalWORMEnabled = bool(wormFlag)
+		globalActiveCred = cred
+		globalConfigEncrypted = true
 	}
 
-	if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
-		globalIsCompressionEnabled = strings.EqualFold(compress, "true")
-	}
-
-	compressExtensions := os.Getenv("MINIO_COMPRESS_EXTENSIONS")
-	compressMimeTypes := os.Getenv("MINIO_COMPRESS_MIMETYPES")
-	if compressExtensions != "" || compressMimeTypes != "" {
-		globalIsEnvCompression = true
-		if compressExtensions != "" {
-			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter))
-			if err != nil {
-				logger.Fatal(err, "Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", extensions)
-			}
-			globalCompressExtensions = extensions
-		}
-		if compressMimeTypes != "" {
-			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter))
-			if err != nil {
-				logger.Fatal(err, "Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", contenttypes)
-			}
-			globalCompressMimeTypes = contenttypes
-		}
+	globalWORMEnabled, err = config.LookupWorm()
+	if err != nil {
+		logger.Fatal(config.ErrInvalidWormValue(err), "Invalid worm configuration")
 	}
 }
+
+func logStartupMessage(msg string) {
+	if globalConsoleSys != nil {
+		globalConsoleSys.Send(msg, string(logger.All))
+	}
+	logger.StartupMessage(msg)
+}
+
+func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
+	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
+		return nil, nil, false, nil
+	}
+
+	if x509Certs, err = config.ParsePublicCertFile(getPublicCertFile()); err != nil {
+		return nil, nil, false, err
+	}
+
+	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), config.LoadX509KeyPair)
+	if err != nil {
+		return nil, nil, false, err
+	}
+
+	secureConn = true
+	return x509Certs, c, secureConn, nil
+}

cmd/config-current_test.go

@@ -19,11 +19,9 @@ package cmd
 import (
 	"context"
 	"os"
-	"path"
 	"testing"
 
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/cmd/config"
 )
 
 func TestServerConfig(t *testing.T) {
@@ -37,19 +35,18 @@ func TestServerConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 
-	if globalServerConfig.GetRegion() != globalMinioDefaultRegion {
-		t.Errorf("Expecting region `us-east-1` found %s", globalServerConfig.GetRegion())
+	if globalServerRegion != globalMinioDefaultRegion {
+		t.Errorf("Expecting region `us-east-1` found %s", globalServerRegion)
 	}
 
 	// Set new region and verify.
-	globalServerConfig.SetRegion("us-west-1")
-	if globalServerConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expecting region `us-west-1` found %s", globalServerConfig.GetRegion())
+	config.SetRegion(globalServerConfig, "us-west-1")
+	region, err := config.LookupRegion(globalServerConfig[config.RegionSubSys][config.Default])
+	if err != nil {
+		t.Fatal(err)
 	}
-
-	// Match version.
-	if globalServerConfig.GetVersion() != serverConfigVersion {
-		t.Errorf("Expecting version %s found %s", globalServerConfig.GetVersion(), serverConfigVersion)
+	if region != "us-west-1" {
+		t.Errorf("Expecting region `us-west-1` found %s", globalServerRegion)
 	}
 
 	if err := saveServerConfig(context.Background(), objLayer, globalServerConfig); err != nil {
@@ -61,306 +58,3 @@ func TestServerConfig(t *testing.T) {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 }
-
-func TestServerConfigWithEnvs(t *testing.T) {
-
-	os.Setenv("MINIO_BROWSER", "off")
-	defer os.Unsetenv("MINIO_BROWSER")
-
-	os.Setenv("MINIO_WORM", "on")
-	defer os.Unsetenv("MINIO_WORM")
-
-	os.Setenv("MINIO_ACCESS_KEY", "minio")
-	defer os.Unsetenv("MINIO_ACCESS_KEY")
-
-	os.Setenv("MINIO_SECRET_KEY", "minio123")
-	defer os.Unsetenv("MINIO_SECRET_KEY")
-
-	os.Setenv("MINIO_REGION", "us-west-1")
-	defer os.Unsetenv("MINIO_REGION")
-
-	os.Setenv("MINIO_DOMAIN", "domain.com")
-	defer os.Unsetenv("MINIO_DOMAIN")
-
-	defer resetGlobalIsEnvs()
-
-	objLayer, fsDir, err := prepareFS()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(fsDir)
-
-	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
-		t.Fatalf("Init Test config failed")
-	}
-
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = objLayer
-	globalObjLayerMutex.Unlock()
-
-	serverHandleEnvVars()
-
-	// Init config
-	initConfig(objLayer)
-
-	// Check if serverConfig has browser disabled
-	if globalIsBrowserEnabled {
-		t.Error("Expected browser to be disabled but it is not")
-	}
-
-	// Check if serverConfig returns WORM config from the env
-	if !globalServerConfig.GetWorm() {
-		t.Error("Expected WORM to be enabled but it is not")
-	}
-
-	// Check if serverConfig has region from the environment
-	if globalServerConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expected region to be \"us-west-1\", found %v", globalServerConfig.GetRegion())
-	}
-
-	// Check if serverConfig has credentials from the environment
-	cred := globalServerConfig.GetCredential()
-
-	if cred.AccessKey != "minio" {
-		t.Errorf("Expected access key to be `minio`, found %s", cred.AccessKey)
-	}
-
-	if cred.SecretKey != "minio123" {
-		t.Errorf("Expected access key to be `minio123`, found %s", cred.SecretKey)
-	}
-
-	// Check if serverConfig has the correct domain
-	if globalDomainNames[0] != "domain.com" {
-		t.Errorf("Expected Domain to be `domain.com`, found " + globalDomainNames[0])
-	}
-}
-
-// Tests config validator..
-func TestValidateConfig(t *testing.T) {
-	objLayer, fsDir, err := prepareFS()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(fsDir)
-
-	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
-		t.Fatalf("Init Test config failed")
-	}
-
-	configPath := path.Join(minioConfigPrefix, minioConfigFile)
-	v := serverConfigVersion
-
-	testCases := []struct {
-		configData string
-		shouldPass bool
-	}{
-		// Test 1 - wrong json
-		{`{`, false},
-
-		// Test 2 - empty json
-		{`{}`, false},
-
-		// Test 3 - wrong config version
-		{`{"version": "10"}`, false},
-
-		// Test 4 - wrong browser parameter
-		{`{"version": "` + v + `", "browser": "foo"}`, false},
-
-		// Test 5 - missing credential
-		{`{"version": "` + v + `", "browser": "on"}`, false},
-
-		// Test 6 - missing secret key
-		{`{"version": "` + v + `", "browser": "on", "credential" : {"accessKey":"minio", "secretKey":""}}`, false},
-
-		// Test 7 - missing region should pass, defaults to 'us-east-1'.
-		{`{"version": "` + v + `", "browser": "on", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, true},
-
-		// Test 8 - missing browser should pass, defaults to 'on'.
-		{`{"version": "` + v + `", "region": "us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, true},
-
-		// Test 9 - success
-		{`{"version": "` + v + `", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, true},
-
-		// Test 10 - duplicated json keys
-		{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
-
-		// Test 11 - Test AMQP
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false, "queueDir": "", "queueLimit": 0}}}}`, false},
-
-		// Test 12 - Test NATS
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "queueDir": "", "queueLimit": 0, "streaming": { "enable": false, "clusterID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
-
-		// Test 13 - Test ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
-
-		// Test 14 - Test Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
-
-		// Test 15 - Test PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "", "queueDir": "", "queueLimit": 0 }}}}`, false},
-
-		// Test 16 - Test Kafka
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "", "queueDir": "", "queueLimit": 0 } }}}`, false},
-
-		// Test 17 - Test Webhook
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "", "queueDir": "", "queueLimit": 0} }}}`, false},
-
-		// Test 18 - Test MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "", "queueDir": "", "queueLimit": 0 }}}}`, false},
-
-		// Test 19 - Test Format for MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
-
-		// Test 20 - Test valid Format for MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
-
-		// Test 21 - Test Format for PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
-
-		// Test 22 - Test valid Format for PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
-
-		// Test 23 - Test Format for ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
-
-		// Test 24 - Test valid Format for ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex", "queueDir": "", "queueLimit": 0 } }}}`, true},
-
-		// Test 25 - Test Format for Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1", "queueDir": "", "queueLimit": 0 } }}}`, false},
-
-		// Test 26 - Test valid Format for Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
-
-		// Test 27 - Test MQTT
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "username": "", "password": "", "queueDir": "", "queueLimit": 0}}}}`, false},
-
-		// Test 28 - Test NSQ
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": "", "queueDir": "", "queueLimit": 0} }}}`, false},
-	}
-
-	for i, testCase := range testCases {
-		if err = saveConfig(context.Background(), objLayer, configPath, []byte(testCase.configData)); err != nil {
-			t.Fatal(err)
-		}
-		_, err = getValidConfig(objLayer)
-		if testCase.shouldPass && err != nil {
-			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, err)
-		}
-		if !testCase.shouldPass && err == nil {
-			t.Errorf("Test %d, should fail but it succeeded.", i+1)
-		}
-	}
-
-}
-
-func TestConfigDiff(t *testing.T) {
-	testCases := []struct {
-		s, t *serverConfig
-		diff string
-	}{
-		// 1
-		{&serverConfig{}, nil, "Given configuration is empty"},
-		// 2
-		{
-			&serverConfig{Credential: auth.Credentials{
-				AccessKey:  "u1",
-				SecretKey:  "p1",
-				Expiration: timeSentinel,
-			}},
-			&serverConfig{Credential: auth.Credentials{
-				AccessKey:  "u1",
-				SecretKey:  "p2",
-				Expiration: timeSentinel,
-			}},
-			"Credential configuration differs",
-		},
-		// 3
-		{&serverConfig{Region: "us-east-1"}, &serverConfig{Region: "us-west-1"}, "Region configuration differs"},
-		// 4
-		{
-			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 6}}},
-			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 4}}},
-			"StorageClass configuration differs",
-		},
-		// 5
-		{
-			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: false}}}},
-			"AMQP Notification configuration differs",
-		},
-		// 6
-		{
-			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: false}}}},
-			"NATS Notification configuration differs",
-		},
-		// 7
-		{
-			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: false}}}},
-			"NSQ Notification configuration differs",
-		},
-		// 8
-		{
-			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: false}}}},
-			"ElasticSearch Notification configuration differs",
-		},
-		// 9
-		{
-			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: false}}}},
-			"Redis Notification configuration differs",
-		},
-		// 10
-		{
-			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: false}}}},
-			"PostgreSQL Notification configuration differs",
-		},
-		// 11
-		{
-			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: false}}}},
-			"Kafka Notification configuration differs",
-		},
-		// 12
-		{
-			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: false}}}},
-			"Webhook Notification configuration differs",
-		},
-		// 13
-		{
-			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: false}}}},
-			"MySQL Notification configuration differs",
-		},
-		// 14
-		{
-			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
-			"MQTT Notification configuration differs",
-		},
-		// 15
-		{
-			&serverConfig{Logger: loggerConfig{
-				Console: loggerConsole{Enabled: true},
-				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address1"}},
-			}},
-			&serverConfig{Logger: loggerConfig{
-				Console: loggerConsole{Enabled: true},
-				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address2"}},
-			}},
-			"Logger configuration differs",
-		},
-	}
-
-	for i, testCase := range testCases {
-		got := testCase.s.ConfigDiff(testCase.t)
-		if got != testCase.diff {
-			t.Errorf("Test %d: got %s expected %s", i+1, got, testCase.diff)
-		}
-	}
-}

cmd/config-encrypted.go

@@ -0,0 +1,367 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"unicode/utf8"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
+	if !server {
+		return nil
+	}
+
+	// If its server mode or nas gateway, migrate the backend.
+	doneCh := make(chan struct{})
+
+	var encrypted bool
+	var err error
+
+	// Migrating Config backend needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	var stop bool
+	for !stop {
+		select {
+		case <-retryTimerCh:
+			if encrypted, err = checkBackendEncrypted(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) {
+					logger.Info("Waiting for config backend to be encrypted..")
+					continue
+				}
+				close(doneCh)
+				return err
+			}
+			stop = true
+		case <-globalOSSignalCh:
+			close(doneCh)
+			return fmt.Errorf("Config encryption process stopped gracefully")
+		}
+	}
+	close(doneCh)
+
+	if encrypted {
+		// backend is encrypted, but credentials are not specified
+		// we shall fail right here. if not proceed forward.
+		if !globalConfigEncrypted || !globalActiveCred.IsValid() {
+			return config.ErrMissingCredentialsBackendEncrypted(nil)
+		}
+	} else {
+		// backend is not yet encrypted, check if encryption of
+		// backend is requested if not return nil and proceed
+		// forward.
+		if !globalConfigEncrypted {
+			return nil
+		}
+		if !globalActiveCred.IsValid() {
+			return errInvalidArgument
+		}
+	}
+
+	activeCredOld, err := getOldCreds()
+	if err != nil {
+		return err
+	}
+
+	doneCh = make(chan struct{})
+	defer close(doneCh)
+
+	retryTimerCh = newRetryTimerSimple(doneCh)
+
+	// Migrating Config backend needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	for {
+		select {
+		case <-retryTimerCh:
+			// Migrate IAM configuration
+			if err = migrateConfigPrefixToEncrypted(objAPI, activeCredOld, encrypted); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for config backend to be encrypted..")
+					continue
+				}
+				return err
+			}
+			return nil
+		case <-globalOSSignalCh:
+			return fmt.Errorf("Config encryption process stopped gracefully")
+		}
+	}
+}
+
+const (
+	backendEncryptedFile = "backend-encrypted"
+)
+
+var (
+	backendEncryptedMigrationIncomplete = []byte("incomplete")
+	backendEncryptedMigrationComplete   = []byte("encrypted")
+)
+
+func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool, error) {
+	data, err := readKeyEtcd(ctx, client, backendEncryptedFile)
+	if err != nil && err != errConfigNotFound {
+		return false, err
+	}
+	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
+}
+
+func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
+	data, err := readConfig(context.Background(), objAPI, backendEncryptedFile)
+	if err != nil && err != errConfigNotFound {
+		return false, err
+	}
+	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
+}
+
+// decryptData - decrypts input data with more that one credentials,
+func decryptData(edata []byte, creds ...auth.Credentials) ([]byte, error) {
+	var err error
+	var data []byte
+	for _, cred := range creds {
+		data, err = madmin.DecryptData(cred.String(), bytes.NewReader(edata))
+		if err != nil {
+			if err == madmin.ErrMaliciousData {
+				continue
+			}
+			return nil, err
+		}
+		break
+	}
+	return data, err
+}
+
+func getOldCreds() (activeCredOld auth.Credentials, err error) {
+	accessKeyOld := env.Get(config.EnvAccessKeyOld, "")
+	secretKeyOld := env.Get(config.EnvSecretKeyOld, "")
+	if accessKeyOld != "" && secretKeyOld != "" {
+		activeCredOld, err = auth.CreateCredentials(accessKeyOld, secretKeyOld)
+		if err != nil {
+			return activeCredOld, err
+		}
+		// Once we have obtained the rotating creds
+		os.Unsetenv(config.EnvAccessKeyOld)
+		os.Unsetenv(config.EnvSecretKeyOld)
+	}
+	return activeCredOld, nil
+}
+
+func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	defer cancel()
+
+	encrypted, err := checkBackendEtcdEncrypted(ctx, client)
+	if err != nil {
+		return err
+	}
+
+	if encrypted {
+		// backend is encrypted, but credentials are not specified
+		// we shall fail right here. if not proceed forward.
+		if !globalConfigEncrypted || !globalActiveCred.IsValid() {
+			return config.ErrMissingCredentialsBackendEncrypted(nil)
+		}
+	} else {
+		// backend is not yet encrypted, check if encryption of
+		// backend is requested if not return nil and proceed
+		// forward.
+		if !globalConfigEncrypted {
+			return nil
+		}
+		if !globalActiveCred.IsValid() {
+			return errInvalidArgument
+		}
+	}
+
+	activeCredOld, err := getOldCreds()
+	if err != nil {
+		return err
+	}
+
+	if encrypted {
+		// No key rotation requested, and backend is
+		// already encrypted. We proceed without migration.
+		if !activeCredOld.IsValid() {
+			return nil
+		}
+
+		// No real reason to rotate if old and new creds are same.
+		if activeCredOld.Equal(globalActiveCred) {
+			return nil
+		}
+
+		logger.Info("Attempting rotation of encrypted IAM users and policies on etcd with newly supplied credentials")
+	} else {
+		logger.Info("Attempting encryption of all IAM users and policies on etcd")
+	}
+
+	r, err := client.Get(ctx, minioConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+	if err != nil {
+		return err
+	}
+
+	if err = saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationIncomplete); err != nil {
+		return err
+	}
+
+	for _, kv := range r.Kvs {
+		var (
+			cdata    []byte
+			cencdata []byte
+		)
+		cdata, err = readKeyEtcd(ctx, client, string(kv.Key))
+		if err != nil {
+			switch err {
+			case errConfigNotFound:
+				// Perhaps not present or someone deleted it.
+				continue
+			}
+			return err
+		}
+
+		var data []byte
+		// Is rotating of creds requested?
+		if activeCredOld.IsValid() {
+			data, err = decryptData(cdata, activeCredOld, globalActiveCred)
+			if err != nil {
+				if err == madmin.ErrMaliciousData {
+					return config.ErrInvalidRotatingCredentialsBackendEncrypted(nil)
+				}
+				return err
+			}
+		} else {
+			data = cdata
+		}
+
+		if !utf8.Valid(data) {
+			return errors.New("config data not in plain-text form")
+		}
+
+		cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
+		if err != nil {
+			return err
+		}
+
+		if err = saveKeyEtcd(ctx, client, string(kv.Key), cencdata); err != nil {
+			return err
+		}
+	}
+	if encrypted && globalActiveCred.IsValid() {
+		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
+	}
+	return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationComplete)
+}
+
+func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Credentials, encrypted bool) error {
+	if encrypted {
+		// No key rotation requested, and backend is
+		// already encrypted. We proceed without migration.
+		if !activeCredOld.IsValid() {
+			return nil
+		}
+
+		// No real reason to rotate if old and new creds are same.
+		if activeCredOld.Equal(globalActiveCred) {
+			return nil
+		}
+		logger.Info("Attempting rotation of encrypted config, IAM users and policies on MinIO with newly supplied credentials")
+	} else {
+		logger.Info("Attempting encryption of all config, IAM users and policies on MinIO backend")
+	}
+
+	err := saveConfig(context.Background(), objAPI, backendEncryptedFile, backendEncryptedMigrationIncomplete)
+	if err != nil {
+		return err
+	}
+
+	var marker string
+	for {
+		res, err := objAPI.ListObjects(context.Background(), minioMetaBucket, minioConfigPrefix, marker, "", maxObjectList)
+		if err != nil {
+			return err
+		}
+		for _, obj := range res.Objects {
+			var (
+				cdata    []byte
+				cencdata []byte
+			)
+
+			cdata, err = readConfig(context.Background(), objAPI, obj.Name)
+			if err != nil {
+				return err
+			}
+
+			var data []byte
+			// Is rotating of creds requested?
+			if activeCredOld.IsValid() {
+				data, err = decryptData(cdata, activeCredOld, globalActiveCred)
+				if err != nil {
+					if err == madmin.ErrMaliciousData {
+						return config.ErrInvalidRotatingCredentialsBackendEncrypted(nil)
+					}
+					return err
+				}
+			} else {
+				data = cdata
+			}
+
+			if !utf8.Valid(data) {
+				return errors.New("config data not in plain-text form")
+			}
+
+			cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
+			if err != nil {
+				return err
+			}
+
+			if err = saveConfig(context.Background(), objAPI, obj.Name, cencdata); err != nil {
+				return err
+			}
+		}
+
+		if !res.IsTruncated {
+			break
+		}
+
+		marker = res.NextMarker
+	}
+
+	if encrypted && globalActiveCred.IsValid() {
+		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
+	}
+
+	return saveConfig(context.Background(), objAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
+}

cmd/config-encrypted_test.go

@@ -0,0 +1,75 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func TestDecryptData(t *testing.T) {
+	cred1 := auth.Credentials{
+		AccessKey: "minio",
+		SecretKey: "minio123",
+	}
+
+	cred2 := auth.Credentials{
+		AccessKey: "minio",
+		SecretKey: "minio1234",
+	}
+
+	data := []byte(`config data`)
+	edata1, err := madmin.EncryptData(cred1.String(), data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	edata2, err := madmin.EncryptData(cred2.String(), data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		edata   []byte
+		creds   []auth.Credentials
+		success bool
+	}{
+		{edata1, []auth.Credentials{cred1, cred2}, true},
+		{edata2, []auth.Credentials{cred1, cred2}, true},
+		{data, []auth.Credentials{cred1, cred2}, false},
+	}
+
+	for _, test := range tests {
+		t.Run("", func(t *testing.T) {
+			ddata, err := decryptData(test.edata, test.creds...)
+			if err != nil && test.success {
+				t.Errorf("Expected success, saw failure %v", err)
+			}
+			if err == nil && !test.success {
+				t.Error("Expected failure, saw success")
+			}
+			if test.success {
+				if !bytes.Equal(ddata, data) {
+					t.Errorf("Expected %s, got %s", string(data), string(ddata))
+				}
+			}
+		})
+	}
+}

cmd/config-migrate.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,20 +17,29 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/cache"
+	"github.com/minio/minio/cmd/config/compress"
+	xldap "github.com/minio/minio/cmd/config/identity/ldap"
+	"github.com/minio/minio/cmd/config/identity/openid"
+	"github.com/minio/minio/cmd/config/notify"
+	"github.com/minio/minio/cmd/config/policy/opa"
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/iam/validator"
+	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
 )
@@ -65,7 +74,7 @@ func migrateConfig() error {
 	// Load only config version information.
 	version, err := GetVersion(getConfigFile())
 	if err != nil {
-		if os.IsNotExist(err) {
+		if os.IsNotExist(err) || os.IsPermission(err) {
 			return nil
 		}
 		return err
@@ -221,7 +230,7 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
-	case serverConfigVersion:
+	case "33":
 		// No migration needed. this always points to current version.
 		err = nil
 	}
@@ -234,10 +243,10 @@ func purgeV1() error {
 
 	cv1 := &configV1{}
 	_, err := Load(configFile, cv1)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘1’. %w", err)
 	}
 	if cv1.Version != "1" {
 		return fmt.Errorf("unrecognized config version ‘%s’", cv1.Version)
@@ -255,10 +264,10 @@ func migrateV2ToV3() error {
 
 	cv2 := &configV2{}
 	_, err := Load(configFile, cv2)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘2’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘2’. %w", err)
 	}
 	if cv2.Version != "2" {
 		return nil
@@ -266,7 +275,7 @@ func migrateV2ToV3() error {
 
 	cred, err := auth.CreateCredentials(cv2.Credentials.AccessKey, cv2.Credentials.SecretKey)
 	if err != nil {
-		return fmt.Errorf("Invalid credential in V2 configuration file. %v", err)
+		return fmt.Errorf("Invalid credential in V2 configuration file. %w", err)
 	}
 
 	srvConfig := &configV3{}
@@ -299,7 +308,7 @@ func migrateV2ToV3() error {
 	srvConfig.Logger.Syslog = slogger
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv2.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv2.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv2.Version, srvConfig.Version)
@@ -314,10 +323,10 @@ func migrateV3ToV4() error {
 
 	cv3 := &configV3{}
 	_, err := Load(configFile, cv3)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘3’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘3’. %w", err)
 	}
 	if cv3.Version != "3" {
 		return nil
@@ -337,7 +346,7 @@ func migrateV3ToV4() error {
 	srvConfig.Logger.Syslog = cv3.Logger.Syslog
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv3.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv3.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv3.Version, srvConfig.Version)
@@ -352,10 +361,10 @@ func migrateV4ToV5() error {
 
 	cv4 := &configV4{}
 	_, err := Load(configFile, cv4)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘4’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘4’. %w", err)
 	}
 	if cv4.Version != "4" {
 		return nil
@@ -378,7 +387,7 @@ func migrateV4ToV5() error {
 	srvConfig.Logger.Redis.Enable = false
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv4.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv4.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv4.Version, srvConfig.Version)
@@ -393,10 +402,10 @@ func migrateV5ToV6() error {
 
 	cv5 := &configV5{}
 	_, err := Load(configFile, cv5)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘5’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘5’. %w", err)
 	}
 	if cv5.Version != "5" {
 		return nil
@@ -438,7 +447,7 @@ func migrateV5ToV6() error {
 
 	if cv5.Logger.ElasticSearch.URL != "" {
 		var url *xnet.URL
-		url, err = xnet.ParseURL(cv5.Logger.ElasticSearch.URL)
+		url, err = xnet.ParseHTTPURL(cv5.Logger.ElasticSearch.URL)
 		if err != nil {
 			return err
 		}
@@ -467,7 +476,7 @@ func migrateV5ToV6() error {
 	}
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv5.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv5.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv5.Version, srvConfig.Version)
@@ -482,10 +491,10 @@ func migrateV6ToV7() error {
 
 	cv6 := &configV6{}
 	_, err := Load(configFile, cv6)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘6’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘6’. %w", err)
 	}
 	if cv6.Version != "6" {
 		return nil
@@ -523,7 +532,7 @@ func migrateV6ToV7() error {
 	}
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv6.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv6.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv6.Version, srvConfig.Version)
@@ -538,10 +547,10 @@ func migrateV7ToV8() error {
 
 	cv7 := &serverConfigV7{}
 	_, err := Load(configFile, cv7)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘7’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘7’. %w", err)
 	}
 	if cv7.Version != "7" {
 		return nil
@@ -586,7 +595,7 @@ func migrateV7ToV8() error {
 	}
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv7.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv7.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv7.Version, srvConfig.Version)
@@ -600,10 +609,10 @@ func migrateV8ToV9() error {
 
 	cv8 := &serverConfigV8{}
 	_, err := Load(configFile, cv8)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘8’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘8’. %w", err)
 	}
 	if cv8.Version != "8" {
 		return nil
@@ -656,7 +665,7 @@ func migrateV8ToV9() error {
 	}
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv8.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv8.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv8.Version, srvConfig.Version)
@@ -670,10 +679,10 @@ func migrateV9ToV10() error {
 
 	cv9 := &serverConfigV9{}
 	_, err := Load(configFile, cv9)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘9’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘9’. %w", err)
 	}
 	if cv9.Version != "9" {
 		return nil
@@ -724,7 +733,7 @@ func migrateV9ToV10() error {
 	}
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv9.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv9.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv9.Version, srvConfig.Version)
@@ -738,10 +747,10 @@ func migrateV10ToV11() error {
 
 	cv10 := &serverConfigV10{}
 	_, err := Load(configFile, cv10)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘10’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘10’. %w", err)
 	}
 	if cv10.Version != "10" {
 		return nil
@@ -795,7 +804,7 @@ func migrateV10ToV11() error {
 	srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv10.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv10.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv10.Version, srvConfig.Version)
@@ -809,10 +818,10 @@ func migrateV11ToV12() error {
 
 	cv11 := &serverConfigV11{}
 	_, err := Load(configFile, cv11)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘11’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘11’. %w", err)
 	}
 	if cv11.Version != "11" {
 		return nil
@@ -893,7 +902,7 @@ func migrateV11ToV12() error {
 	}
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv11.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv11.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv11.Version, srvConfig.Version)
@@ -906,10 +915,10 @@ func migrateV12ToV13() error {
 
 	cv12 := &serverConfigV12{}
 	_, err := Load(configFile, cv12)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘12’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘12’. %w", err)
 	}
 	if cv12.Version != "12" {
 		return nil
@@ -973,7 +982,7 @@ func migrateV12ToV13() error {
 	srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv12.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv12.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv12.Version, srvConfig.Version)
@@ -986,10 +995,10 @@ func migrateV13ToV14() error {
 
 	cv13 := &serverConfigV13{}
 	_, err := Load(configFile, cv13)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘13’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘13’. %w", err)
 	}
 	if cv13.Version != "13" {
 		return nil
@@ -1058,7 +1067,7 @@ func migrateV13ToV14() error {
 	srvConfig.Browser = true
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv13.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv13.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv13.Version, srvConfig.Version)
@@ -1071,10 +1080,10 @@ func migrateV14ToV15() error {
 
 	cv14 := &serverConfigV14{}
 	_, err := Load(configFile, cv14)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘14’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘14’. %w", err)
 	}
 	if cv14.Version != "14" {
 		return nil
@@ -1147,7 +1156,7 @@ func migrateV14ToV15() error {
 	srvConfig.Browser = cv14.Browser
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv14.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv14.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv14.Version, srvConfig.Version)
@@ -1161,10 +1170,10 @@ func migrateV15ToV16() error {
 
 	cv15 := &serverConfigV15{}
 	_, err := Load(configFile, cv15)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘15’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘15’. %w", err)
 	}
 	if cv15.Version != "15" {
 		return nil
@@ -1237,7 +1246,7 @@ func migrateV15ToV16() error {
 	srvConfig.Browser = cv15.Browser
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv15.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv15.Version, srvConfig.Version)
@@ -1251,10 +1260,10 @@ func migrateV16ToV17() error {
 
 	cv16 := &serverConfigV16{}
 	_, err := Load(configFile, cv16)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘16’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘16’. %w", err)
 	}
 	if cv16.Version != "16" {
 		return nil
@@ -1358,7 +1367,7 @@ func migrateV16ToV17() error {
 	srvConfig.Browser = cv16.Browser
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv16.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv16.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv16.Version, srvConfig.Version)
@@ -1372,10 +1381,10 @@ func migrateV17ToV18() error {
 
 	cv17 := &serverConfigV17{}
 	_, err := Load(configFile, cv17)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘17’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘17’. %w", err)
 	}
 	if cv17.Version != "17" {
 		return nil
@@ -1462,7 +1471,7 @@ func migrateV17ToV18() error {
 	srvConfig.Browser = cv17.Browser
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv17.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv17.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv17.Version, srvConfig.Version)
@@ -1474,10 +1483,10 @@ func migrateV18ToV19() error {
 
 	cv18 := &serverConfigV18{}
 	_, err := Load(configFile, cv18)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘18’. %w", err)
 	}
 	if cv18.Version != "18" {
 		return nil
@@ -1568,7 +1577,7 @@ func migrateV18ToV19() error {
 	srvConfig.Browser = cv18.Browser
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv18.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv18.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version)
@@ -1580,10 +1589,10 @@ func migrateV19ToV20() error {
 
 	cv19 := &serverConfigV19{}
 	_, err := Load(configFile, cv19)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘18’. %w", err)
 	}
 	if cv19.Version != "19" {
 		return nil
@@ -1673,7 +1682,7 @@ func migrateV19ToV20() error {
 	srvConfig.Browser = cv19.Browser
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv19.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version)
@@ -1685,10 +1694,10 @@ func migrateV20ToV21() error {
 
 	cv20 := &serverConfigV20{}
 	_, err := Load(configFile, cv20)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘20’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘20’. %w", err)
 	}
 	if cv20.Version != "20" {
 		return nil
@@ -1777,7 +1786,7 @@ func migrateV20ToV21() error {
 	srvConfig.Domain = cv20.Domain
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv20.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv20.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv20.Version, srvConfig.Version)
@@ -1789,10 +1798,10 @@ func migrateV21ToV22() error {
 
 	cv21 := &serverConfigV21{}
 	_, err := Load(configFile, cv21)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘21’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘21’. %w", err)
 	}
 	if cv21.Version != "21" {
 		return nil
@@ -1881,7 +1890,7 @@ func migrateV21ToV22() error {
 	srvConfig.Domain = cv21.Domain
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv21.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv21.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv21.Version, srvConfig.Version)
@@ -1893,10 +1902,10 @@ func migrateV22ToV23() error {
 
 	cv22 := &serverConfigV22{}
 	_, err := Load(configFile, cv22)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘22’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘22’. %w", err)
 	}
 	if cv22.Version != "22" {
 		return nil
@@ -1991,10 +2000,10 @@ func migrateV22ToV23() error {
 	// Init cache config.For future migration, Cache config needs to be copied over from previous version.
 	srvConfig.Cache.Drives = []string{}
 	srvConfig.Cache.Exclude = []string{}
-	srvConfig.Cache.Expiry = globalCacheExpiry
+	srvConfig.Cache.Expiry = 90
 
 	if err = Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv22.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv22.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv22.Version, srvConfig.Version)
@@ -2006,10 +2015,10 @@ func migrateV23ToV24() error {
 
 	cv23 := &serverConfigV23{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv23)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘23’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘23’. %w", err)
 	}
 	if cv23.Version != "23" {
 		return nil
@@ -2107,7 +2116,7 @@ func migrateV23ToV24() error {
 	srvConfig.Cache.Expiry = cv23.Cache.Expiry
 
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv23.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv23.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv23.Version, srvConfig.Version)
@@ -2119,10 +2128,10 @@ func migrateV24ToV25() error {
 
 	cv24 := &serverConfigV24{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv24)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘24’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘24’. %w", err)
 	}
 	if cv24.Version != "24" {
 		return nil
@@ -2225,7 +2234,7 @@ func migrateV24ToV25() error {
 	srvConfig.Cache.Expiry = cv24.Cache.Expiry
 
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv24.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv24.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv24.Version, srvConfig.Version)
@@ -2237,10 +2246,10 @@ func migrateV25ToV26() error {
 
 	cv25 := &serverConfigV25{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘25’. %v", err)
+		return fmt.Errorf("Unable to load config version ‘25’. %w", err)
 	}
 	if cv25.Version != "25" {
 		return nil
@@ -2341,10 +2350,10 @@ func migrateV25ToV26() error {
 	srvConfig.Cache.Expiry = cv25.Cache.Expiry
 
 	// Add predefined value to new server config.
-	srvConfig.Cache.MaxUse = globalCacheMaxUse
+	srvConfig.Cache.MaxUse = 80
 
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv25.Version, srvConfig.Version, err)
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv25.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv25.Version, srvConfig.Version)
@@ -2359,10 +2368,10 @@ func migrateV26ToV27() error {
 	// in the new `logger` field
 	srvConfig := &serverConfigV27{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %v", err)
+		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 
 	if srvConfig.Version != "26" {
@@ -2373,11 +2382,11 @@ func migrateV26ToV27() error {
 	// Enable console logging by default to avoid breaking users
 	// current deployments
 	srvConfig.Logger.Console.Enabled = true
-	srvConfig.Logger.HTTP = make(map[string]loggerHTTP)
-	srvConfig.Logger.HTTP["1"] = loggerHTTP{}
+	srvConfig.Logger.HTTP = make(map[string]logger.HTTP)
+	srvConfig.Logger.HTTP["1"] = logger.HTTP{}
 
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %v", err)
+		return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %w", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "26", "27")
@@ -2392,10 +2401,10 @@ func migrateV27ToV28() error {
 
 	srvConfig := &serverConfigV28{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %v", err)
+		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 
 	if srvConfig.Version != "27" {
@@ -2405,7 +2414,7 @@ func migrateV27ToV28() error {
 	srvConfig.Version = "28"
 	srvConfig.KMS = crypto.KMSConfig{}
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
+		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %w", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
@@ -2418,11 +2427,6 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 
-	// Verify if config was already available in .minio.sys in which case, nothing more to be done.
-	if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
-		return err
-	}
-
 	defer func() {
 		if err == nil {
 			if globalEtcdClient != nil {
@@ -2435,17 +2439,6 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		}
 	}()
 
-	transactionConfigFile := configFile + ".transaction"
-
-	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
-	// and configFile, take a transaction lock to avoid data race between readConfig()
-	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, transactionConfigFile)
-	if err = objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
-
 	// Verify if backend already has the file (after holding lock)
 	if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
 		return err
@@ -2456,17 +2449,20 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		getConfigFile() + ".deprecated",
 		configFile,
 	}
-	var config = &serverConfig{}
+	var config = &serverConfigV27{}
 	for _, cfgFile := range configFiles {
 		if _, err = Load(cfgFile, config); err != nil {
-			if !os.IsNotExist(err) {
+			if !os.IsNotExist(err) && !os.IsPermission(err) {
 				return err
 			}
 			continue
 		}
 		break
 	}
-	if os.IsNotExist(err) {
+	if os.IsPermission(err) {
+		logger.Info("Older config found but not readable %s, proceeding to initialize new config anyways", err)
+	}
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		// Initialize the server config, if no config exists.
 		return newSrvConfig(objAPI)
 	}
@@ -2475,10 +2471,11 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 
 // Migrates '.minio.sys/config.json' to v33.
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
+	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 
 	// Check if the config version is latest, if not migrate.
-	ok, _, err := checkConfigVersion(objAPI, configFile, serverConfigVersion)
+	ok, _, err := checkConfigVersion(objAPI, configFile, "33")
 	if err != nil {
 		return err
 	}
@@ -2486,18 +2483,6 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
 		return nil
 	}
 
-	// Construct path to config.json for the given bucket.
-	transactionConfigFile := configFile + ".transaction"
-
-	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
-	// and configFile, take a transaction lock to avoid data race between readConfig()
-	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, transactionConfigFile)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
-
 	if err := migrateV27ToV28MinioSys(objAPI); err != nil {
 		return err
 	}
@@ -2522,6 +2507,16 @@ func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (
 		return false, nil, err
 	}
 
+	if globalConfigEncrypted {
+		data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
+		if err != nil {
+			if err == madmin.ErrMaliciousData {
+				return false, nil, config.ErrInvalidCredentialsBackendEncrypted(nil)
+			}
+			return false, nil, err
+		}
+	}
+
 	var versionConfig struct {
 		Version string `json:"version"`
 	}
@@ -2539,7 +2534,7 @@ func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %v", err)
+		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
@@ -2553,13 +2548,8 @@ func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
 	cfg.Version = "28"
 	cfg.KMS = crypto.KMSConfig{}
 
-	data, err = json.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-
-	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
+	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %w", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
@@ -2573,7 +2563,7 @@ func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %v", err)
+		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
@@ -2585,13 +2575,8 @@ func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
 	}
 
 	cfg.Version = "29"
-	data, err = json.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-
-	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
+	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %w", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "28", "29")
@@ -2605,7 +2590,7 @@ func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %v", err)
+		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
@@ -2619,16 +2604,11 @@ func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
 	cfg.Version = "30"
 	// Init compression config.For future migration, Compression config needs to be copied over from previous version.
 	cfg.Compression.Enabled = false
-	cfg.Compression.Extensions = globalCompressExtensions
-	cfg.Compression.MimeTypes = globalCompressMimeTypes
+	cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
+	cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
 
-	data, err = json.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-
-	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %v", err)
+	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %w", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
@@ -2642,7 +2622,7 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %v", err)
+		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
@@ -2654,21 +2634,16 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
 	}
 
 	cfg.Version = "31"
-	cfg.OpenID.JWKS = validator.JWKSArgs{
-		URL: &xnet.URL{},
-	}
-	cfg.Policy.OPA = iampolicy.OpaArgs{
+	cfg.OpenID = openid.Config{}
+	cfg.OpenID.JWKS.URL = &xnet.URL{}
+
+	cfg.Policy.OPA = opa.Args{
 		URL:       &xnet.URL{},
 		AuthToken: "",
 	}
 
-	data, err = json.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-
-	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %v", err)
+	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %w", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "30", "31")
@@ -2682,7 +2657,7 @@ func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %v", err)
+		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
@@ -2697,13 +2672,8 @@ func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
 	cfg.Notify.NSQ = make(map[string]target.NSQArgs)
 	cfg.Notify.NSQ["1"] = target.NSQArgs{}
 
-	data, err = json.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-
-	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %v", err)
+	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %w", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "31", "32")
@@ -2717,7 +2687,7 @@ func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %v", err)
+		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
@@ -2730,15 +2700,88 @@ func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
 
 	cfg.Version = "33"
 
-	data, err = json.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-
-	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
-		return fmt.Errorf("Failed to migrate config from  32  to  33 . %v", err)
+	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+		return fmt.Errorf("Failed to migrate config from '32' to '33' . %w", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "32", "33")
 	return nil
 }
+
+func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	// Check if the config version is latest, if not migrate.
+	ok, data, err := checkConfigVersion(objAPI, configFile, "33")
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return nil
+	}
+
+	cfg := &serverConfigV33{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
+	}
+
+	newCfg := newServerConfig()
+
+	config.SetCredentials(newCfg, cfg.Credential)
+	config.SetRegion(newCfg, cfg.Region)
+
+	storageclass.SetStorageClass(newCfg, cfg.StorageClass)
+
+	for k, loggerArgs := range cfg.Logger.HTTP {
+		logger.SetLoggerHTTP(newCfg, k, loggerArgs)
+	}
+	for k, auditArgs := range cfg.Logger.Audit {
+		logger.SetLoggerHTTPAudit(newCfg, k, auditArgs)
+	}
+
+	crypto.SetKMSConfig(newCfg, cfg.KMS)
+	xldap.SetIdentityLDAP(newCfg, cfg.LDAPServerConfig)
+	openid.SetIdentityOpenID(newCfg, cfg.OpenID)
+	opa.SetPolicyOPAConfig(newCfg, cfg.Policy.OPA)
+	cache.SetCacheConfig(newCfg, cfg.Cache)
+	compress.SetCompressionConfig(newCfg, cfg.Compression)
+
+	for k, args := range cfg.Notify.AMQP {
+		notify.SetNotifyAMQP(newCfg, k, args)
+	}
+	for k, args := range cfg.Notify.Elasticsearch {
+		notify.SetNotifyES(newCfg, k, args)
+	}
+	for k, args := range cfg.Notify.Kafka {
+		notify.SetNotifyKafka(newCfg, k, args)
+	}
+	for k, args := range cfg.Notify.MQTT {
+		notify.SetNotifyMQTT(newCfg, k, args)
+	}
+	for k, args := range cfg.Notify.MySQL {
+		notify.SetNotifyMySQL(newCfg, k, args)
+	}
+	for k, args := range cfg.Notify.NATS {
+		notify.SetNotifyNATS(newCfg, k, args)
+	}
+	for k, args := range cfg.Notify.NSQ {
+		notify.SetNotifyNSQ(newCfg, k, args)
+	}
+	for k, args := range cfg.Notify.PostgreSQL {
+		notify.SetNotifyPostgres(newCfg, k, args)
+	}
+	for k, args := range cfg.Notify.Redis {
+		notify.SetNotifyRedis(newCfg, k, args)
+	}
+	for k, args := range cfg.Notify.Webhook {
+		notify.SetNotifyWebhook(newCfg, k, args)
+	}
+
+	if err = saveServerConfig(context.Background(), objAPI, newCfg); err != nil {
+		return err
+	}
+
+	logger.Info("Configuration file %s migrated from version '%s' to new KV format successfully.",
+		configFile, "33")
+	return nil
+}

cmd/config-migrate_test.go

@@ -21,6 +21,8 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+
+	"github.com/minio/minio/cmd/config"
 )
 
 // Test if config v1 is purged
@@ -208,24 +210,24 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
+	if err := migrateMinioSysConfigToKV(objLayer); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
 	// Initialize server config and check again if everything is fine
 	if err := loadConfig(objLayer); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 
-	// Check the version number in the upgraded config file
-	expectedVersion := serverConfigVersion
-	if globalServerConfig.Version != expectedVersion {
-		t.Fatalf("Expect version "+expectedVersion+", found: %v", globalServerConfig.Version)
-	}
-
 	// Check if accessKey and secretKey are not altered during migration
-	if globalServerConfig.Credential.AccessKey != accessKey {
-		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, globalServerConfig.Credential.AccessKey)
+	caccessKey := globalServerConfig[config.CredentialsSubSys][config.Default].Get(config.AccessKey)
+	if caccessKey != accessKey {
+		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, caccessKey)
 	}
 
-	if globalServerConfig.Credential.SecretKey != secretKey {
-		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, globalServerConfig.Credential.SecretKey)
+	csecretKey := globalServerConfig[config.CredentialsSubSys][config.Default].Get(config.SecretKey)
+	if csecretKey != secretKey {
+		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, csecretKey)
 	}
 }
 

cmd/config-versions.go

@@ -19,11 +19,18 @@ package cmd
 import (
 	"sync"
 
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/cache"
+	"github.com/minio/minio/cmd/config/compress"
+	xldap "github.com/minio/minio/cmd/config/identity/ldap"
+	"github.com/minio/minio/cmd/config/identity/openid"
+	"github.com/minio/minio/cmd/config/notify"
+	"github.com/minio/minio/cmd/config/policy/opa"
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -265,7 +272,7 @@ type serverConfigV7 struct {
 	Notify notifierV1 `json:"notify"`
 }
 
-// serverConfigV8 server configuration version '8'. Adds NATS notifier
+// serverConfigV8 server configuration version '8'. Adds NATS notify.Config
 // configuration.
 type serverConfigV8 struct {
 	Version string `json:"version"`
@@ -282,7 +289,7 @@ type serverConfigV8 struct {
 }
 
 // serverConfigV9 server configuration version '9'. Adds PostgreSQL
-// notifier configuration.
+// notify.Config configuration.
 type serverConfigV9 struct {
 	Version string `json:"version"`
 
@@ -400,7 +407,7 @@ type serverConfigV14 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -417,7 +424,7 @@ type serverConfigV15 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -455,7 +462,7 @@ type serverConfigV16 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -474,7 +481,7 @@ type serverConfigV17 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -493,7 +500,7 @@ type serverConfigV18 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -511,7 +518,7 @@ type serverConfigV19 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -529,7 +536,7 @@ type serverConfigV20 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Additional error logging configuration.
@@ -547,7 +554,7 @@ type serverConfigV21 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Notification queue configuration.
@@ -556,43 +563,37 @@ type serverConfigV21 struct {
 
 // serverConfigV22 is just like version '21' with added support
 // for StorageClass.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV22 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV23 is just like version '22' with addition of cache field.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV23 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
@@ -600,23 +601,20 @@ type serverConfigV23 struct {
 
 // serverConfigV24 is just like version '23', we had to revert
 // the changes which were made in 6fb06045028b7a57c37c60a612c8e50735279ab4
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV24 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
@@ -624,9 +622,6 @@ type serverConfigV24 struct {
 
 // serverConfigV25 is just like version '24', stores additionally
 // worm variable.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV25 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -635,22 +630,22 @@ type serverConfigV25 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
-	Worm       BoolFlag         `json:"worm"`
+	Browser    config.BoolFlag  `json:"browser"`
+	Worm       config.BoolFlag  `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV26 is just like version '25', stores additionally
-// cache max use value in 'CacheConfig'.
+// cache max use value in 'cache.Config'.
 type serverConfigV26 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -659,39 +654,22 @@ type serverConfigV26 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
-	Worm       BoolFlag         `json:"worm"`
+	Browser    config.BoolFlag  `json:"browser"`
+	Worm       config.BoolFlag  `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
 
-type loggerConsole struct {
-	Enabled bool `json:"enabled"`
-}
-
-type loggerHTTP struct {
-	Enabled  bool   `json:"enabled"`
-	Endpoint string `json:"endpoint"`
-}
-
-type loggerConfig struct {
-	Console loggerConsole         `json:"console"`
-	HTTP    map[string]loggerHTTP `json:"http"`
-}
-
 // serverConfigV27 is just like version '26', stores additionally
 // the logger field
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV27 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -700,28 +678,25 @@ type serverConfigV27 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
-	Worm       BoolFlag         `json:"worm"`
+	Browser    config.BoolFlag  `json:"browser"`
+	Worm       config.BoolFlag  `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 }
 
 // serverConfigV28 is just like version '27', additionally
 // storing KMS config
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV28 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -730,13 +705,13 @@ type serverConfigV28 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
@@ -745,19 +720,12 @@ type serverConfigV28 struct {
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 }
 
 // serverConfigV29 is just like version '28'.
 type serverConfigV29 serverConfigV28
 
-// compressionConfig represents the compression settings.
-type compressionConfig struct {
-	Enabled    bool     `json:"enabled"`
-	Extensions []string `json:"extensions"`
-	MimeTypes  []string `json:"mime-types"`
-}
-
 // serverConfigV30 is just like version '29', stores additionally
 // extensions and mimetypes fields for compression.
 type serverConfigV30 struct {
@@ -766,13 +734,13 @@ type serverConfigV30 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
@@ -781,10 +749,10 @@ type serverConfigV30 struct {
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 }
 
 // serverConfigV31 is just like version '30', with OPA and OpenID configuration.
@@ -794,13 +762,13 @@ type serverConfigV31 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
@@ -809,39 +777,23 @@ type serverConfigV31 struct {
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 
 	// OpenID configuration
-	OpenID struct {
-		// JWKS validator config.
-		JWKS validator.JWKSArgs `json:"jwks"`
-	} `json:"openid"`
+	OpenID openid.Config `json:"openid"`
 
 	// External policy enforcements.
 	Policy struct {
 		// OPA configuration.
-		OPA iampolicy.OpaArgs `json:"opa"`
+		OPA opa.Args `json:"opa"`
 
 		// Add new external policy enforcements here.
 	} `json:"policy"`
 }
 
-type notifier struct {
-	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
-	Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
-	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
-	MQTT          map[string]target.MQTTArgs          `json:"mqtt"`
-	MySQL         map[string]target.MySQLArgs         `json:"mysql"`
-	NATS          map[string]target.NATSArgs          `json:"nats"`
-	NSQ           map[string]target.NSQArgs           `json:"nsq"`
-	PostgreSQL    map[string]target.PostgreSQLArgs    `json:"postgresql"`
-	Redis         map[string]target.RedisArgs         `json:"redis"`
-	Webhook       map[string]target.WebhookArgs       `json:"webhook"`
-}
-
 // serverConfigV32 is just like version '31' with added nsq notifer.
 type serverConfigV32 struct {
 	Version string `json:"version"`
@@ -849,36 +801,33 @@ type serverConfigV32 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notify.Config `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 
 	// OpenID configuration
-	OpenID struct {
-		// JWKS validator config.
-		JWKS validator.JWKSArgs `json:"jwks"`
-	} `json:"openid"`
+	OpenID openid.Config `json:"openid"`
 
 	// External policy enforcements.
 	Policy struct {
 		// OPA configuration.
-		OPA iampolicy.OpaArgs `json:"opa"`
+		OPA opa.Args `json:"opa"`
 
 		// Add new external policy enforcements here.
 	} `json:"policy"`
@@ -893,37 +842,36 @@ type serverConfigV33 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notify.Config `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 
 	// OpenID configuration
-	OpenID struct {
-		// JWKS validator config.
-		JWKS validator.JWKSArgs `json:"jwks"`
-	} `json:"openid"`
+	OpenID openid.Config `json:"openid"`
 
 	// External policy enforcements.
 	Policy struct {
 		// OPA configuration.
-		OPA iampolicy.OpaArgs `json:"opa"`
+		OPA opa.Args `json:"opa"`
 
 		// Add new external policy enforcements here.
 	} `json:"policy"`
+
+	LDAPServerConfig xldap.Config `json:"ldapserverconfig"`
 }

cmd/config.go

@@ -20,74 +20,157 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"path"
-	"runtime"
+	"sort"
 	"strings"
 	"time"
 
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/quick"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 const (
 	minioConfigPrefix = "config"
 
+	kvPrefix = ".kv"
+
+	// Captures all the previous SetKV operations and allows rollback.
+	minioConfigHistoryPrefix = minioConfigPrefix + "/history"
+
 	// MinIO configuration file.
 	minioConfigFile = "config.json"
 
-	// MinIO backup file
+	// MinIO configuration backup file
 	minioConfigBackupFile = minioConfigFile + ".backup"
 )
 
-func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverConfig) error {
-	if err := quick.CheckData(config); err != nil {
-		return err
+func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData bool, count int) (
+	[]madmin.ConfigHistoryEntry, error) {
+
+	var configHistory []madmin.ConfigHistoryEntry
+
+	// List all kvs
+	marker := ""
+	for {
+		res, err := objAPI.ListObjects(ctx, minioMetaBucket, minioConfigHistoryPrefix, marker, "", maxObjectList)
+		if err != nil {
+			return nil, err
+		}
+		for _, obj := range res.Objects {
+			cfgEntry := madmin.ConfigHistoryEntry{
+				RestoreID:  strings.TrimSuffix(path.Base(obj.Name), kvPrefix),
+				CreateTime: obj.ModTime, // ModTime is createTime for config history entries.
+			}
+			if withData {
+				data, err := readConfig(ctx, objAPI, obj.Name)
+				if err != nil {
+					return nil, err
+				}
+				if globalConfigEncrypted {
+					data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
+					if err != nil {
+						return nil, err
+					}
+				}
+				cfgEntry.Data = string(data)
+			}
+			configHistory = append(configHistory, cfgEntry)
+			count--
+			if count == 0 {
+				break
+			}
+		}
+		if !res.IsTruncated {
+			// We are done here
+			break
+		}
+		marker = res.NextMarker
+	}
+	sort.Slice(configHistory, func(i, j int) bool {
+		return configHistory[i].CreateTime.Before(configHistory[j].CreateTime)
+	})
+	return configHistory, nil
+}
+
+func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error {
+	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
+	return objAPI.DeleteObject(ctx, minioMetaBucket, historyFile)
+}
+
+func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) ([]byte, error) {
+	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
+	data, err := readConfig(ctx, objAPI, historyFile)
+	if err != nil {
+		return nil, err
 	}
 
-	data, err := json.MarshalIndent(config, "", "\t")
+	if globalConfigEncrypted {
+		data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
+	}
+
+	return data, err
+}
+
+func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
+	uuidKV := mustGetUUID() + kvPrefix
+	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV)
+
+	var err error
+	if globalConfigEncrypted {
+		kv, err = madmin.EncryptData(globalActiveCred.String(), kv)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Save the new config KV settings into the history path.
+	return saveConfig(ctx, objAPI, historyFile, kv)
+}
+
+func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config interface{}) error {
+	data, err := json.Marshal(config)
 	if err != nil {
 		return err
 	}
 
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	// Create a backup of the current config
-	oldData, err := readConfig(ctx, objAPI, configFile)
-	if err == nil {
-		backupConfigFile := path.Join(minioConfigPrefix, minioConfigBackupFile)
-		if err = saveConfig(ctx, objAPI, backupConfigFile, oldData); err != nil {
-			return err
-		}
-	} else {
-		if err != errConfigNotFound {
+	if globalConfigEncrypted {
+		data, err = madmin.EncryptData(globalActiveCred.String(), data)
+		if err != nil {
 			return err
 		}
 	}
 
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	// Save the new config in the std config path
 	return saveConfig(ctx, objAPI, configFile, data)
 }
 
-func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
+func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, error) {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
+		// Config not found for some reason, allow things to continue
+		// by initializing a new fresh config in safe mode.
+		if err == errConfigNotFound && globalSafeMode {
+			return newServerConfig(), nil
+		}
 		return nil, err
 	}
 
-	if runtime.GOOS == "windows" {
-		configData = bytes.Replace(configData, []byte("\r\n"), []byte("\n"), -1)
+	if globalConfigEncrypted {
+		configData, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(configData))
+		if err != nil {
+			if err == madmin.ErrMaliciousData {
+				return nil, config.ErrInvalidCredentialsBackendEncrypted(nil)
+			}
+			return nil, err
+		}
 	}
 
-	if err = quick.CheckDuplicateKeys(string(configData)); err != nil {
-		return nil, err
-	}
-
-	var config = &serverConfig{}
-	if err = json.Unmarshal(configData, config); err != nil {
-		return nil, err
-	}
-
-	if err = quick.CheckData(config); err != nil {
+	var config = config.New()
+	if err = json.Unmarshal(configData, &config); err != nil {
 		return nil, err
 	}
 
@@ -136,18 +219,24 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	//    of the object layer.
 	//  - Write quorum not met when upgrading configuration
 	//    version is needed.
-	for range newRetryTimerSimple(doneCh) {
-		if err := initConfig(objAPI); err != nil {
-			if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-				logger.Info("Waiting for configuration to be initialized..")
-				continue
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case <-retryTimerCh:
+			if err := initConfig(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for configuration to be initialized..")
+					continue
+				}
+				return err
 			}
-			return err
+			return nil
+		case <-globalOSSignalCh:
+			return fmt.Errorf("Initializing config sub-system gracefully stopped")
 		}
-		break
 	}
-	return nil
 }
 
 // NewConfigSys - creates new config system object.
@@ -181,5 +270,11 @@ func initConfig(objAPI ObjectLayer) error {
 		return err
 	}
 
+	// Migrates backend '<export_path>/.minio.sys/config/config.json' to
+	// latest config format.
+	if err := migrateMinioSysConfigToKV(objAPI); err != nil {
+		return err
+	}
+
 	return loadConfig(objAPI)
 }

cmd/config/bool-flag.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,11 +14,13 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"encoding/json"
 	"fmt"
+	"strconv"
+	"strings"
 )
 
 // BoolFlag - wrapper bool type.
@@ -54,16 +56,35 @@ func (bf *BoolFlag) UnmarshalJSON(data []byte) (err error) {
 	return err
 }
 
+// FormatBool prints stringified version of boolean.
+func FormatBool(b bool) string {
+	if b {
+		return "on"
+	}
+	return "off"
+}
+
+// ParseBool returns the boolean value represented by the string.
+// It accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False.
+// Any other value returns an error.
+func ParseBool(str string) (bool, error) {
+	switch str {
+	case "1", "t", "T", "true", "TRUE", "True", "on", "ON", "On":
+		return true, nil
+	case "0", "f", "F", "false", "FALSE", "False", "off", "OFF", "Off":
+		return false, nil
+	}
+	if strings.EqualFold(str, "enabled") {
+		return true, nil
+	}
+	if strings.EqualFold(str, "disabled") {
+		return false, nil
+	}
+	return false, fmt.Errorf("ParseBool: parsing '%s': %s", str, strconv.ErrSyntax)
+}
+
 // ParseBoolFlag - parses string into BoolFlag.
 func ParseBoolFlag(s string) (bf BoolFlag, err error) {
-	switch s {
-	case "on":
-		bf = true
-	case "off":
-		bf = false
-	default:
-		err = fmt.Errorf("invalid value ‘%s’ for BoolFlag", s)
-	}
-
-	return bf, err
+	b, err := ParseBool(s)
+	return BoolFlag(b), err
 }

cmd/config/bool-flag_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,10 +14,9 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
-	"errors"
 	"testing"
 )
 
@@ -68,33 +67,29 @@ func TestBoolFlagUnmarshalJSON(t *testing.T) {
 	testCases := []struct {
 		data           []byte
 		expectedResult BoolFlag
-		expectedErr    error
+		expectedErr    bool
 	}{
-		{[]byte(`{}`), BoolFlag(false), errors.New("json: cannot unmarshal object into Go value of type string")},
-		{[]byte(`["on"]`), BoolFlag(false), errors.New("json: cannot unmarshal array into Go value of type string")},
-		{[]byte(`"junk"`), BoolFlag(false), errors.New("invalid value ‘junk’ for BoolFlag")},
-		{[]byte(`"true"`), BoolFlag(false), errors.New("invalid value ‘true’ for BoolFlag")},
-		{[]byte(`"false"`), BoolFlag(false), errors.New("invalid value ‘false’ for BoolFlag")},
-		{[]byte(`"ON"`), BoolFlag(false), errors.New("invalid value ‘ON’ for BoolFlag")},
-		{[]byte(`"OFF"`), BoolFlag(false), errors.New("invalid value ‘OFF’ for BoolFlag")},
-		{[]byte(`""`), BoolFlag(true), nil},
-		{[]byte(`"on"`), BoolFlag(true), nil},
-		{[]byte(`"off"`), BoolFlag(false), nil},
+		{[]byte(`{}`), BoolFlag(false), true},
+		{[]byte(`["on"]`), BoolFlag(false), true},
+		{[]byte(`"junk"`), BoolFlag(false), true},
+		{[]byte(`""`), BoolFlag(true), false},
+		{[]byte(`"on"`), BoolFlag(true), false},
+		{[]byte(`"off"`), BoolFlag(false), false},
+		{[]byte(`"true"`), BoolFlag(true), false},
+		{[]byte(`"false"`), BoolFlag(false), false},
+		{[]byte(`"ON"`), BoolFlag(true), false},
+		{[]byte(`"OFF"`), BoolFlag(false), false},
 	}
 
 	for _, testCase := range testCases {
 		var flag BoolFlag
 		err := (&flag).UnmarshalJSON(testCase.data)
-		if testCase.expectedErr == nil {
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
-			}
-		} else if err == nil {
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		if !testCase.expectedErr && err != nil {
+			t.Fatalf("error: expected = <nil>, got = %v", err)
+		}
+		if testCase.expectedErr && err == nil {
+			t.Fatalf("error: expected error, got = <nil>")
 		}
-
 		if err == nil && testCase.expectedResult != flag {
 			t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, flag)
 		}
@@ -106,30 +101,26 @@ func TestParseBoolFlag(t *testing.T) {
 	testCases := []struct {
 		flagStr        string
 		expectedResult BoolFlag
-		expectedErr    error
+		expectedErr    bool
 	}{
-		{"", BoolFlag(false), errors.New("invalid value ‘’ for BoolFlag")},
-		{"junk", BoolFlag(false), errors.New("invalid value ‘junk’ for BoolFlag")},
-		{"true", BoolFlag(false), errors.New("invalid value ‘true’ for BoolFlag")},
-		{"false", BoolFlag(false), errors.New("invalid value ‘false’ for BoolFlag")},
-		{"ON", BoolFlag(false), errors.New("invalid value ‘ON’ for BoolFlag")},
-		{"OFF", BoolFlag(false), errors.New("invalid value ‘OFF’ for BoolFlag")},
-		{"on", BoolFlag(true), nil},
-		{"off", BoolFlag(false), nil},
+		{"", BoolFlag(false), true},
+		{"junk", BoolFlag(false), true},
+		{"true", BoolFlag(true), false},
+		{"false", BoolFlag(false), false},
+		{"ON", BoolFlag(true), false},
+		{"OFF", BoolFlag(false), false},
+		{"on", BoolFlag(true), false},
+		{"off", BoolFlag(false), false},
 	}
 
 	for _, testCase := range testCases {
 		bf, err := ParseBoolFlag(testCase.flagStr)
-		if testCase.expectedErr == nil {
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
-			}
-		} else if err == nil {
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		if !testCase.expectedErr && err != nil {
+			t.Fatalf("error: expected = <nil>, got = %v", err)
+		}
+		if testCase.expectedErr && err == nil {
+			t.Fatalf("error: expected error, got = <nil>")
 		}
-
 		if err == nil && testCase.expectedResult != bf {
 			t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, bf)
 		}

cmd/config/cache/config.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package cache
 
 import (
 	"encoding/json"
@@ -22,21 +22,24 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/ellipses"
 )
 
-// CacheConfig represents cache config settings
-type CacheConfig struct {
+// Config represents cache config settings
+type Config struct {
+	Enabled bool     `json:"-"`
 	Drives  []string `json:"drives"`
 	Expiry  int      `json:"expiry"`
 	MaxUse  int      `json:"maxuse"`
+	Quota   int      `json:"quota"`
 	Exclude []string `json:"exclude"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
 // json entries for CacheConfig.
-func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
-	type Alias CacheConfig
+func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
+	type Alias Config
 	var _cfg = &struct {
 		*Alias
 	}{
@@ -54,22 +57,30 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
 		return errors.New("config max use value should not be null or negative")
 	}
 
-	if _, err = parseCacheDrives(_cfg.Drives); err != nil {
-		return err
-	}
-	if _, err = parseCacheExcludes(_cfg.Exclude); err != nil {
-		return err
+	if _cfg.Quota < 0 {
+		return errors.New("config quota value should not be null or negative")
 	}
+
 	return nil
 }
 
 // Parses given cacheDrivesEnv and returns a list of cache drives.
-func parseCacheDrives(drives []string) ([]string, error) {
+func parseCacheDrives(drives string) ([]string, error) {
+	var drivesSlice []string
 	if len(drives) == 0 {
-		return drives, nil
+		return drivesSlice, nil
 	}
+
+	drivesSlice = strings.Split(drives, cacheDelimiterLegacy)
+	if len(drivesSlice) == 1 && drivesSlice[0] == drives {
+		drivesSlice = strings.Split(drives, cacheDelimiter)
+	}
+
 	var endpoints []string
-	for _, d := range drives {
+	for _, d := range drivesSlice {
+		if len(d) == 0 {
+			return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir cannot be an empty path")
+		}
 		if ellipses.HasEllipses(d) {
 			s, err := parseCacheDrivePaths(d)
 			if err != nil {
@@ -83,7 +94,7 @@ func parseCacheDrives(drives []string) ([]string, error) {
 
 	for _, d := range endpoints {
 		if !filepath.IsAbs(d) {
-			return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
+			return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
 		}
 	}
 	return endpoints, nil
@@ -93,7 +104,7 @@ func parseCacheDrives(drives []string) ([]string, error) {
 func parseCacheDrivePaths(arg string) (ep []string, err error) {
 	patterns, perr := ellipses.FindEllipsesPatterns(arg)
 	if perr != nil {
-		return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
+		return []string{}, config.ErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
 	}
 
 	for _, lbls := range patterns.Expand() {
@@ -104,14 +115,25 @@ func parseCacheDrivePaths(arg string) (ep []string, err error) {
 }
 
 // Parses given cacheExcludesEnv and returns a list of cache exclude patterns.
-func parseCacheExcludes(excludes []string) ([]string, error) {
-	for _, e := range excludes {
+func parseCacheExcludes(excludes string) ([]string, error) {
+	var excludesSlice []string
+	if len(excludes) == 0 {
+		return excludesSlice, nil
+	}
+
+	excludesSlice = strings.Split(excludes, cacheDelimiterLegacy)
+	if len(excludesSlice) == 1 && excludesSlice[0] == excludes {
+		excludesSlice = strings.Split(excludes, cacheDelimiter)
+	}
+
+	for _, e := range excludesSlice {
 		if len(e) == 0 {
-			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
+			return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
 		}
-		if hasPrefix(e, SlashSeparator) {
-			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
+		if strings.HasPrefix(e, "/") {
+			return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
 		}
 	}
-	return excludes, nil
+
+	return excludesSlice, nil
 }

cmd/config/cache/config_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,12 +14,11 @@
  * limitations under the License.
  */
 
-package cmd
+package cache
 
 import (
 	"reflect"
 	"runtime"
-	"strings"
 	"testing"
 )
 
@@ -30,12 +29,16 @@ func TestParseCacheDrives(t *testing.T) {
 		expectedPatterns []string
 		success          bool
 	}{
-		// valid input
+		// Invalid input
 
 		{"bucket1/*;*.png;images/trip/barcelona/*", []string{}, false},
 		{"bucket1", []string{}, false},
+		{";;;", []string{}, false},
+		{",;,;,;", []string{}, false},
 	}
-	if runtime.GOOS == globalWindowsOSName {
+
+	// Valid inputs
+	if runtime.GOOS == "windows" {
 		testCases = append(testCases, struct {
 			driveStr         string
 			expectedPatterns []string
@@ -57,6 +60,11 @@ func TestParseCacheDrives(t *testing.T) {
 			expectedPatterns []string
 			success          bool
 		}{"/home/drive1;/home/drive2;/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive1,/home/drive2,/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
 		testCases = append(testCases, struct {
 			driveStr         string
 			expectedPatterns []string
@@ -69,7 +77,7 @@ func TestParseCacheDrives(t *testing.T) {
 		}{"/home/drive{1..3}", []string{}, false})
 	}
 	for i, testCase := range testCases {
-		drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheEnvDelimiter))
+		drives, err := parseCacheDrives(testCase.driveStr)
 		if err != nil && testCase.success {
 			t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
 		}
@@ -91,14 +99,19 @@ func TestParseCacheExclude(t *testing.T) {
 		expectedPatterns []string
 		success          bool
 	}{
-		// valid input
+		// Invalid input
 		{"/home/drive1;/home/drive2;/home/drive3", []string{}, false},
+		{"/", []string{}, false},
+		{";;;", []string{}, false},
+
+		// valid input
 		{"bucket1/*;*.png;images/trip/barcelona/*", []string{"bucket1/*", "*.png", "images/trip/barcelona/*"}, true},
+		{"bucket1/*,*.png,images/trip/barcelona/*", []string{"bucket1/*", "*.png", "images/trip/barcelona/*"}, true},
 		{"bucket1", []string{"bucket1"}, true},
 	}
 
 	for i, testCase := range testCases {
-		excludes, err := parseCacheExcludes(strings.Split(testCase.excludeStr, cacheEnvDelimiter))
+		excludes, err := parseCacheExcludes(testCase.excludeStr)
 		if err != nil && testCase.success {
 			t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
 		}

cmd/config/cache/help.go

@@ -0,0 +1,54 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cache
+
+import "github.com/minio/minio/cmd/config"
+
+// Help template for caching feature.
+var (
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         Drives,
+			Description: `comma separated mountpoints e.g. "/optane1,/optane2"`,
+			Type:        "csv",
+		},
+		config.HelpKV{
+			Key:         Expiry,
+			Description: `cache expiry duration in days e.g. "90"`,
+			Optional:    true,
+			Type:        "number",
+		},
+		config.HelpKV{
+			Key:         Quota,
+			Description: `limit cache drive usage in percentage e.g. "90"`,
+			Optional:    true,
+			Type:        "number",
+		},
+		config.HelpKV{
+			Key:         Exclude,
+			Description: `comma separated wildcard exclusion patterns e.g. "bucket/*.tmp,*.exe"`,
+			Optional:    true,
+			Type:        "csv",
+		},
+		config.HelpKV{
+			Key:         config.Comment,
+			Description: config.DefaultComment,
+			Optional:    true,
+			Type:        "sentence",
+		},
+	}
+)

cmd/config/cache/legacy.go

@@ -0,0 +1,54 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cache
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+)
+
+const (
+	cacheDelimiterLegacy = ";"
+)
+
+// SetCacheConfig - One time migration code needed, for migrating from older config to new for Cache.
+func SetCacheConfig(s config.Config, cfg Config) {
+	if len(cfg.Drives) == 0 {
+		// Do not save cache if no settings available.
+		return
+	}
+	s[config.CacheSubSys][config.Default] = config.KVS{
+		config.KV{
+			Key:   Drives,
+			Value: strings.Join(cfg.Drives, cacheDelimiter),
+		},
+		config.KV{
+			Key:   Exclude,
+			Value: strings.Join(cfg.Exclude, cacheDelimiter),
+		},
+		config.KV{
+			Key:   Expiry,
+			Value: fmt.Sprintf("%d", cfg.Expiry),
+		},
+		config.KV{
+			Key:   Quota,
+			Value: fmt.Sprintf("%d", cfg.MaxUse),
+		},
+	}
+}

cmd/config/cache/lookup.go

@@ -0,0 +1,140 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cache
+
+import (
+	"errors"
+	"strconv"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Cache ENVs
+const (
+	Drives  = "drives"
+	Exclude = "exclude"
+	Expiry  = "expiry"
+	MaxUse  = "maxuse"
+	Quota   = "quota"
+
+	EnvCacheDrives              = "MINIO_CACHE_DRIVES"
+	EnvCacheExclude             = "MINIO_CACHE_EXCLUDE"
+	EnvCacheExpiry              = "MINIO_CACHE_EXPIRY"
+	EnvCacheMaxUse              = "MINIO_CACHE_MAXUSE"
+	EnvCacheQuota               = "MINIO_CACHE_QUOTA"
+	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
+
+	DefaultExpiry = "90"
+	DefaultQuota  = "80"
+)
+
+// DefaultKVS - default KV settings for caching.
+var (
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   Drives,
+			Value: "",
+		},
+		config.KV{
+			Key:   Exclude,
+			Value: "",
+		},
+		config.KV{
+			Key:   Expiry,
+			Value: DefaultExpiry,
+		},
+		config.KV{
+			Key:   Quota,
+			Value: DefaultQuota,
+		},
+	}
+)
+
+const (
+	cacheDelimiter = ","
+)
+
+// Enabled returns if cache is enabled.
+func Enabled(kvs config.KVS) bool {
+	drives := kvs.Get(Drives)
+	return drives != ""
+}
+
+// LookupConfig - extracts cache configuration provided by environment
+// variables and merge them with provided CacheConfiguration.
+func LookupConfig(kvs config.KVS) (Config, error) {
+	cfg := Config{}
+
+	if err := config.CheckValidKeys(config.CacheSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+
+	drives := env.Get(EnvCacheDrives, kvs.Get(Drives))
+	if len(drives) == 0 {
+		return cfg, nil
+	}
+
+	var err error
+	cfg.Drives, err = parseCacheDrives(drives)
+	if err != nil {
+		return cfg, err
+	}
+
+	cfg.Enabled = true
+	if excludes := env.Get(EnvCacheExclude, kvs.Get(Exclude)); excludes != "" {
+		cfg.Exclude, err = parseCacheExcludes(excludes)
+		if err != nil {
+			return cfg, err
+		}
+	}
+
+	if expiryStr := env.Get(EnvCacheExpiry, kvs.Get(Expiry)); expiryStr != "" {
+		cfg.Expiry, err = strconv.Atoi(expiryStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheExpiryValue(err)
+		}
+	}
+
+	if maxUseStr := env.Get(EnvCacheMaxUse, kvs.Get(MaxUse)); maxUseStr != "" {
+		cfg.MaxUse, err = strconv.Atoi(maxUseStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheQuota(err)
+		}
+		// maxUse should be a valid percentage.
+		if cfg.MaxUse < 0 || cfg.MaxUse > 100 {
+			err := errors.New("config max use value should not be null or negative")
+			return cfg, config.ErrInvalidCacheQuota(err)
+		}
+		cfg.Quota = cfg.MaxUse
+	}
+
+	if quotaStr := env.Get(EnvCacheQuota, kvs.Get(Quota)); quotaStr != "" {
+		cfg.Quota, err = strconv.Atoi(quotaStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheQuota(err)
+		}
+		// quota should be a valid percentage.
+		if cfg.Quota < 0 || cfg.Quota > 100 {
+			err := errors.New("config quota value should not be null or negative")
+			return cfg, config.ErrInvalidCacheQuota(err)
+		}
+		cfg.MaxUse = cfg.Quota
+	}
+
+	return cfg, nil
+}

cmd/config/certs.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"bytes"
@@ -25,16 +25,18 @@ import (
 	"encoding/pem"
 	"io/ioutil"
 	"os"
+	"path"
 
-	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/env"
 )
 
-// TLSPrivateKeyPassword is the environment variable which contains the password used
+// EnvCertPassword is the environment variable which contains the password used
 // to decrypt the TLS private key. It must be set if the TLS private key is
 // password protected.
-const TLSPrivateKeyPassword = "MINIO_CERT_PASSWD"
+const EnvCertPassword = "MINIO_CERT_PASSWD"
 
-func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
+// ParsePublicCertFile - parses public cert into its *x509.Certificate equivalent.
+func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
 	// Read certificate file.
 	var data []byte
 	if data, err = ioutil.ReadFile(certFile); err != nil {
@@ -49,25 +51,27 @@ func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 	for len(current) > 0 {
 		var pemBlock *pem.Block
 		if pemBlock, current = pem.Decode(current); pemBlock == nil {
-			return nil, uiErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
+			return nil, ErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
 		}
 
 		var x509Cert *x509.Certificate
 		if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
-			return nil, uiErrSSLUnexpectedData(err)
+			return nil, ErrSSLUnexpectedData(err)
 		}
 
 		x509Certs = append(x509Certs, x509Cert)
 	}
 
 	if len(x509Certs) == 0 {
-		return nil, uiErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
+		return nil, ErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
 	}
 
 	return x509Certs, nil
 }
 
-func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
+// GetRootCAs - returns all the root CAs into certPool
+// at the input certsCADir
+func GetRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	rootCAs, _ := x509.SystemCertPool()
 	if rootCAs == nil {
 		// In some systems (like Windows) system cert pool is
@@ -76,86 +80,70 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 		rootCAs = x509.NewCertPool()
 	}
 
-	fis, err := readDir(certsCAsDir)
+	fis, err := ioutil.ReadDir(certsCAsDir)
 	if err != nil {
-		if err == errFileNotFound {
-			err = nil // Return success if CA's directory is missing.
+		if os.IsNotExist(err) || os.IsPermission(err) {
+			// Return success if CA's directory is missing or permission denied.
+			err = nil
 		}
 		return rootCAs, err
 	}
 
 	// Load all custom CA files.
 	for _, fi := range fis {
-		// Skip all directories.
-		if hasSuffix(fi, SlashSeparator) {
-			continue
-		}
-		caCert, err := ioutil.ReadFile(pathJoin(certsCAsDir, fi))
+		caCert, err := ioutil.ReadFile(path.Join(certsCAsDir, fi.Name()))
 		if err != nil {
-			return rootCAs, err
+			// ignore files which are not readable.
+			continue
 		}
 		rootCAs.AppendCertsFromPEM(caCert)
 	}
 	return rootCAs, nil
 }
 
-// load an X509 key pair (private key , certificate) from the provided
-// paths. The private key may be encrypted and is decrypted using the
-// ENV_VAR: MINIO_CERT_PASSWD.
-func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
+// LoadX509KeyPair - load an X509 key pair (private key , certificate)
+// from the provided paths. The private key may be encrypted and is
+// decrypted using the ENV_VAR: MINIO_CERT_PASSWD.
+func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	certPEMBlock, err := ioutil.ReadFile(certFile)
 	if err != nil {
-		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
+		return tls.Certificate{}, ErrSSLUnexpectedError(err)
 	}
 	keyPEMBlock, err := ioutil.ReadFile(keyFile)
 	if err != nil {
-		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
+		return tls.Certificate{}, ErrSSLUnexpectedError(err)
 	}
 	key, rest := pem.Decode(keyPEMBlock)
 	if len(rest) > 0 {
-		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
+		return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
 	}
 	if x509.IsEncryptedPEMBlock(key) {
-		password, ok := os.LookupEnv(TLSPrivateKeyPassword)
-		if !ok {
-			return tls.Certificate{}, uiErrSSLNoPassword(nil)
+		password := env.Get(EnvCertPassword, "")
+		if len(password) == 0 {
+			return tls.Certificate{}, ErrSSLNoPassword(nil)
 		}
 		decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
 		if decErr != nil {
-			return tls.Certificate{}, uiErrSSLWrongPassword(decErr)
+			return tls.Certificate{}, ErrSSLWrongPassword(decErr)
 		}
 		keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
 	}
 	cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
 	if err != nil {
-		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg(err.Error())
+		return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg(err.Error())
 	}
 	// Ensure that the private key is not a P-384 or P-521 EC key.
 	// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
 	if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
 		if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
-			if name := pub.Params().Name; name == "P-384" || name == "P-521" { // unfortunately there is no cleaner way to check
-				return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
+			switch pub.Params().Name {
+			case "P-384":
+				fallthrough
+			case "P-521":
+				// unfortunately there is no cleaner way to check
+				return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", pub.Params().Name)
 			}
 		}
 	}
 	return cert, nil
 }
-
-func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
-	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
-		return nil, nil, false, nil
-	}
-
-	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
-		return nil, nil, false, err
-	}
-
-	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
-	if err != nil {
-		return nil, nil, false, err
-	}
-
-	secureConn = true
-	return x509Certs, c, secureConn, nil
-}

cmd/config/certs_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"fmt"
@@ -176,7 +176,7 @@ M9ofSEt/bdRD
 	}
 
 	for _, testCase := range testCases {
-		certs, err := parsePublicCertFile(testCase.certFile)
+		certs, err := ParsePublicCertFile(testCase.certFile)
 
 		if testCase.expectedErr == nil {
 			if err != nil {
@@ -234,7 +234,7 @@ func TestGetRootCAs(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
-		_, err := getRootCAs(testCase.certCAsDir)
+		_, err := GetRootCAs(testCase.certCAsDir)
 
 		if testCase.expectedErr == nil {
 			if err != nil {
@@ -260,11 +260,11 @@ func TestLoadX509KeyPair(t *testing.T) {
 			t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err)
 		}
 
-		os.Unsetenv(TLSPrivateKeyPassword)
+		os.Unsetenv(EnvCertPassword)
 		if testCase.password != "" {
-			os.Setenv(TLSPrivateKeyPassword, testCase.password)
+			os.Setenv(EnvCertPassword, testCase.password)
 		}
-		_, err = loadX509KeyPair(certificate, privateKey)
+		_, err = LoadX509KeyPair(certificate, privateKey)
 		if err != nil && !testCase.shouldFail {
 			t.Errorf("Test %d: test should succeed but it failed: %v", i, err)
 		}

cmd/config/compress/compress.go

@@ -0,0 +1,132 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package compress
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Config represents the compression settings.
+type Config struct {
+	Enabled    bool     `json:"enabled"`
+	Extensions []string `json:"extensions"`
+	MimeTypes  []string `json:"mime-types"`
+}
+
+// Compression environment variables
+const (
+	Extensions = "extensions"
+	MimeTypes  = "mime_types"
+
+	EnvCompressState      = "MINIO_COMPRESS_ENABLE"
+	EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressMimeTypes  = "MINIO_COMPRESS_MIME_TYPES"
+
+	// Include-list for compression.
+	DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
+	DefaultMimeTypes  = "text/*,application/json,application/xml"
+)
+
+// DefaultKVS - default KV config for compression settings
+var (
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   config.Enable,
+			Value: config.EnableOff,
+		},
+		config.KV{
+			Key:   Extensions,
+			Value: DefaultExtensions,
+		},
+		config.KV{
+			Key:   MimeTypes,
+			Value: DefaultMimeTypes,
+		},
+	}
+)
+
+// Parses the given compression exclude list `extensions` or `content-types`.
+func parseCompressIncludes(include string) ([]string, error) {
+	includes := strings.Split(include, config.ValueSeparator)
+	for _, e := range includes {
+		if len(e) == 0 {
+			return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be empty")
+		}
+		if e == "/" {
+			return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be '/'")
+		}
+	}
+	return includes, nil
+}
+
+// LookupConfig - lookup compression config.
+func LookupConfig(kvs config.KVS) (Config, error) {
+	var err error
+	cfg := Config{}
+	if err = config.CheckValidKeys(config.CompressionSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+
+	compress := env.Get(EnvCompress, "")
+	if compress == "" {
+		compress = env.Get(EnvCompressState, kvs.Get(config.Enable))
+	}
+	cfg.Enabled, err = config.ParseBool(compress)
+	if err != nil {
+		// Parsing failures happen due to empty KVS, ignore it.
+		if kvs.Empty() {
+			return cfg, nil
+		}
+		return cfg, err
+	}
+	if !cfg.Enabled {
+		return cfg, nil
+	}
+
+	compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
+	compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
+	compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))
+	if compressExtensions != "" || compressMimeTypes != "" || compressMimeTypesLegacy != "" {
+		if compressExtensions != "" {
+			extensions, err := parseCompressIncludes(compressExtensions)
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
+			}
+			cfg.Extensions = extensions
+		}
+		if compressMimeTypes != "" {
+			mimeTypes, err := parseCompressIncludes(compressMimeTypes)
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
+			}
+			cfg.MimeTypes = mimeTypes
+		}
+		if compressMimeTypesLegacy != "" {
+			mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy)
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes)
+			}
+			cfg.MimeTypes = mimeTypes
+		}
+	}
+
+	return cfg, nil
+}

cmd/config/compress/compress_test.go

@@ -0,0 +1,57 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package compress
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseCompressIncludes(t *testing.T) {
+	testCases := []struct {
+		str              string
+		expectedPatterns []string
+		success          bool
+	}{
+		// invalid input
+		{",,,", []string{}, false},
+		{"", []string{}, false},
+		{",", []string{}, false},
+		{"/", []string{}, false},
+		{"text/*,/", []string{}, false},
+
+		// valid input
+		{".txt,.log", []string{".txt", ".log"}, true},
+		{"text/*,application/json", []string{"text/*", "application/json"}, true},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCase.str, func(t *testing.T) {
+			gotPatterns, err := parseCompressIncludes(testCase.str)
+			if !testCase.success && err == nil {
+				t.Error("expected failure but success instead")
+			}
+			if testCase.success && err != nil {
+				t.Errorf("expected success but failed instead %s", err)
+			}
+			if testCase.success && !reflect.DeepEqual(testCase.expectedPatterns, gotPatterns) {
+				t.Errorf("expected patterns %s but got %s", testCase.expectedPatterns, gotPatterns)
+			}
+		})
+	}
+}

cmd/config/compress/help.go

@@ -0,0 +1,43 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package compress
+
+import "github.com/minio/minio/cmd/config"
+
+// Help template for compress feature.
+var (
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         Extensions,
+			Description: `comma separated file extensions e.g. ".txt,.log,.csv"`,
+			Optional:    true,
+			Type:        "csv",
+		},
+		config.HelpKV{
+			Key:         MimeTypes,
+			Description: `comma separated wildcard mime-types e.g. "text/*,application/json,application/xml"`,
+			Optional:    true,
+			Type:        "csv",
+		},
+		config.HelpKV{
+			Key:         config.Comment,
+			Description: config.DefaultComment,
+			Optional:    true,
+			Type:        "sentence",
+		},
+	}
+)

cmd/config/compress/legacy.go

@@ -0,0 +1,51 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package compress
+
+import (
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+)
+
+// Legacy envs.
+const (
+	EnvCompress                = "MINIO_COMPRESS"
+	EnvCompressMimeTypesLegacy = "MINIO_COMPRESS_MIMETYPES"
+)
+
+// SetCompressionConfig - One time migration code needed, for migrating from older config to new for Compression.
+func SetCompressionConfig(s config.Config, cfg Config) {
+	if !cfg.Enabled {
+		// No need to save disabled settings in new config.
+		return
+	}
+	s[config.CompressionSubSys][config.Default] = config.KVS{
+		config.KV{
+			Key:   config.Enable,
+			Value: config.EnableOn,
+		},
+		config.KV{
+			Key:   Extensions,
+			Value: strings.Join(cfg.Extensions, config.ValueSeparator),
+		},
+		config.KV{
+			Key:   MimeTypes,
+			Value: strings.Join(cfg.MimeTypes, config.ValueSeparator),
+		},
+	}
+}

cmd/config/config.go

@@ -0,0 +1,661 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package config
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+// Error config error type
+type Error struct {
+	Err string
+}
+
+// Errorf - formats according to a format specifier and returns
+// the string as a value that satisfies error of type config.Error
+func Errorf(format string, a ...interface{}) error {
+	return Error{Err: fmt.Sprintf(format, a...)}
+}
+
+func (e Error) Error() string {
+	return e.Err
+}
+
+// Default keys
+const (
+	Default = madmin.Default
+	Enable  = madmin.EnableKey
+	Comment = madmin.CommentKey
+
+	// Enable values
+	EnableOn  = madmin.EnableOn
+	EnableOff = madmin.EnableOff
+
+	RegionName = "name"
+	AccessKey  = "access_key"
+	SecretKey  = "secret_key"
+)
+
+// Top level config constants.
+const (
+	CredentialsSubSys    = "credentials"
+	PolicyOPASubSys      = "policy_opa"
+	IdentityOpenIDSubSys = "identity_openid"
+	IdentityLDAPSubSys   = "identity_ldap"
+	CacheSubSys          = "cache"
+	RegionSubSys         = "region"
+	EtcdSubSys           = "etcd"
+	StorageClassSubSys   = "storage_class"
+	CompressionSubSys    = "compression"
+	KmsVaultSubSys       = "kms_vault"
+	KmsKesSubSys         = "kms_kes"
+	LoggerWebhookSubSys  = "logger_webhook"
+	AuditWebhookSubSys   = "audit_webhook"
+
+	// Add new constants here if you add new fields to config.
+)
+
+// Notification config constants.
+const (
+	NotifyKafkaSubSys    = "notify_kafka"
+	NotifyMQTTSubSys     = "notify_mqtt"
+	NotifyMySQLSubSys    = "notify_mysql"
+	NotifyNATSSubSys     = "notify_nats"
+	NotifyNSQSubSys      = "notify_nsq"
+	NotifyESSubSys       = "notify_elasticsearch"
+	NotifyAMQPSubSys     = "notify_amqp"
+	NotifyPostgresSubSys = "notify_postgres"
+	NotifyRedisSubSys    = "notify_redis"
+	NotifyWebhookSubSys  = "notify_webhook"
+
+	// Add new constants here if you add new fields to config.
+)
+
+// SubSystems - all supported sub-systems
+var SubSystems = set.CreateStringSet([]string{
+	CredentialsSubSys,
+	RegionSubSys,
+	EtcdSubSys,
+	CacheSubSys,
+	StorageClassSubSys,
+	CompressionSubSys,
+	KmsVaultSubSys,
+	KmsKesSubSys,
+	LoggerWebhookSubSys,
+	AuditWebhookSubSys,
+	PolicyOPASubSys,
+	IdentityLDAPSubSys,
+	IdentityOpenIDSubSys,
+	NotifyAMQPSubSys,
+	NotifyESSubSys,
+	NotifyKafkaSubSys,
+	NotifyMQTTSubSys,
+	NotifyMySQLSubSys,
+	NotifyNATSSubSys,
+	NotifyNSQSubSys,
+	NotifyPostgresSubSys,
+	NotifyRedisSubSys,
+	NotifyWebhookSubSys,
+}...)
+
+// SubSystemsSingleTargets - subsystems which only support single target.
+var SubSystemsSingleTargets = set.CreateStringSet([]string{
+	CredentialsSubSys,
+	RegionSubSys,
+	EtcdSubSys,
+	CacheSubSys,
+	StorageClassSubSys,
+	CompressionSubSys,
+	KmsVaultSubSys,
+	KmsKesSubSys,
+	PolicyOPASubSys,
+	IdentityLDAPSubSys,
+	IdentityOpenIDSubSys,
+}...)
+
+// Constant separators
+const (
+	SubSystemSeparator = madmin.SubSystemSeparator
+	KvSeparator        = madmin.KvSeparator
+	KvSpaceSeparator   = madmin.KvSpaceSeparator
+	KvComment          = madmin.KvComment
+	KvNewline          = madmin.KvNewline
+	KvDoubleQuote      = madmin.KvDoubleQuote
+	KvSingleQuote      = madmin.KvSingleQuote
+
+	// Env prefix used for all envs in MinIO
+	EnvPrefix        = "MINIO_"
+	EnvWordDelimiter = `_`
+)
+
+// DefaultKVS - default kvs for all sub-systems
+var DefaultKVS map[string]KVS
+
+// RegisterDefaultKVS - this function saves input kvsMap
+// globally, this should be called only once preferably
+// during `init()`.
+func RegisterDefaultKVS(kvsMap map[string]KVS) {
+	DefaultKVS = map[string]KVS{}
+	for subSys, kvs := range kvsMap {
+		DefaultKVS[subSys] = kvs
+	}
+}
+
+// HelpSubSysMap - help for all individual KVS for each sub-systems
+// also carries a special empty sub-system which dumps
+// help for each sub-system key.
+var HelpSubSysMap map[string]HelpKVS
+
+// RegisterHelpSubSys - this function saves
+// input help KVS for each sub-system globally,
+// this function should be called only once
+// preferably in during `init()`.
+func RegisterHelpSubSys(helpKVSMap map[string]HelpKVS) {
+	HelpSubSysMap = map[string]HelpKVS{}
+	for subSys, hkvs := range helpKVSMap {
+		HelpSubSysMap[subSys] = hkvs
+	}
+}
+
+// KV - is a shorthand of each key value.
+type KV struct {
+	Key   string `json:"key"`
+	Value string `json:"value"`
+}
+
+// KVS - is a shorthand for some wrapper functions
+// to operate on list of key values.
+type KVS []KV
+
+// Empty - return if kv is empty
+func (kvs KVS) Empty() bool {
+	return len(kvs) == 0
+}
+
+func (kvs KVS) String() string {
+	var s strings.Builder
+	for _, kv := range kvs {
+		// Do not need to print if state is on
+		if kv.Key == Enable && kv.Value == EnableOn {
+			continue
+		}
+		s.WriteString(kv.Key)
+		s.WriteString(KvSeparator)
+		spc := madmin.HasSpace(kv.Value)
+		if spc {
+			s.WriteString(KvDoubleQuote)
+		}
+		s.WriteString(kv.Value)
+		if spc {
+			s.WriteString(KvDoubleQuote)
+		}
+		s.WriteString(KvSpaceSeparator)
+	}
+	return s.String()
+}
+
+// Set sets a value, if not sets a default value.
+func (kvs *KVS) Set(key, value string) {
+	for i, kv := range *kvs {
+		if kv.Key == key {
+			(*kvs)[i] = KV{
+				Key:   key,
+				Value: value,
+			}
+			return
+		}
+	}
+	*kvs = append(*kvs, KV{
+		Key:   key,
+		Value: value,
+	})
+}
+
+// Get - returns the value of a key, if not found returns empty.
+func (kvs KVS) Get(key string) string {
+	v, ok := kvs.Lookup(key)
+	if ok {
+		return v
+	}
+	return ""
+}
+
+// Lookup - lookup a key in a list of KVS
+func (kvs KVS) Lookup(key string) (string, bool) {
+	for _, kv := range kvs {
+		if kv.Key == key {
+			return kv.Value, true
+		}
+	}
+	return "", false
+}
+
+// Config - MinIO server config structure.
+type Config map[string]map[string]KVS
+
+// DelFrom - deletes all keys in the input reader.
+func (c Config) DelFrom(r io.Reader) error {
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		// Skip any empty lines, or comment like characters
+		text := scanner.Text()
+		if text == "" || strings.HasPrefix(text, KvComment) {
+			continue
+		}
+		if err := c.DelKVS(text); err != nil {
+			return err
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ReadFrom - implements io.ReaderFrom interface
+func (c Config) ReadFrom(r io.Reader) (int64, error) {
+	var n int
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		// Skip any empty lines, or comment like characters
+		text := scanner.Text()
+		if text == "" || strings.HasPrefix(text, KvComment) {
+			continue
+		}
+		if err := c.SetKVS(text, DefaultKVS); err != nil {
+			return 0, err
+		}
+		n += len(text)
+	}
+	if err := scanner.Err(); err != nil {
+		return 0, err
+	}
+	return int64(n), nil
+}
+
+type configWriteTo struct {
+	Config
+	filterByKey string
+}
+
+// NewConfigWriteTo - returns a struct which
+// allows for serializing the config/kv struct
+// to a io.WriterTo
+func NewConfigWriteTo(cfg Config, key string) io.WriterTo {
+	return &configWriteTo{Config: cfg, filterByKey: key}
+}
+
+// WriteTo - implements io.WriterTo interface implementation for config.
+func (c *configWriteTo) WriteTo(w io.Writer) (int64, error) {
+	kvsTargets, err := c.GetKVS(c.filterByKey, DefaultKVS)
+	if err != nil {
+		return 0, err
+	}
+	var n int
+	for _, target := range kvsTargets {
+		m1, _ := w.Write([]byte(target.SubSystem))
+		m2, _ := w.Write([]byte(KvSpaceSeparator))
+		m3, _ := w.Write([]byte(target.KVS.String()))
+		if len(kvsTargets) > 1 {
+			m4, _ := w.Write([]byte(KvNewline))
+			n += m1 + m2 + m3 + m4
+		} else {
+			n += m1 + m2 + m3
+		}
+	}
+	return int64(n), nil
+}
+
+// Default KV configs for worm and region
+var (
+	DefaultCredentialKVS = KVS{
+		KV{
+			Key:   AccessKey,
+			Value: auth.DefaultAccessKey,
+		},
+		KV{
+			Key:   SecretKey,
+			Value: auth.DefaultSecretKey,
+		},
+	}
+
+	DefaultRegionKVS = KVS{
+		KV{
+			Key:   RegionName,
+			Value: "",
+		},
+	}
+)
+
+// LookupCreds - lookup credentials from config.
+func LookupCreds(kv KVS) (auth.Credentials, error) {
+	if err := CheckValidKeys(CredentialsSubSys, kv, DefaultCredentialKVS); err != nil {
+		return auth.Credentials{}, err
+	}
+	accessKey := kv.Get(AccessKey)
+	secretKey := kv.Get(SecretKey)
+	if accessKey == "" || secretKey == "" {
+		accessKey = auth.DefaultAccessKey
+		secretKey = auth.DefaultSecretKey
+	}
+	return auth.CreateCredentials(accessKey, secretKey)
+}
+
+var validRegionRegex = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9-_-]+$")
+
+// LookupRegion - get current region.
+func LookupRegion(kv KVS) (string, error) {
+	if err := CheckValidKeys(RegionSubSys, kv, DefaultRegionKVS); err != nil {
+		return "", err
+	}
+	region := env.Get(EnvRegion, "")
+	if region == "" {
+		region = env.Get(EnvRegionName, kv.Get(RegionName))
+	}
+	if region != "" {
+		if validRegionRegex.MatchString(region) {
+			return region, nil
+		}
+		return "", Errorf(
+			"region '%s' is invalid, expected simple characters such as [us-east-1, myregion...]",
+			region)
+	}
+	return "", nil
+}
+
+// CheckValidKeys - checks if inputs KVS has the necessary keys,
+// returns error if it find extra or superflous keys.
+func CheckValidKeys(subSys string, kv KVS, validKVS KVS) error {
+	nkv := KVS{}
+	for _, kv := range kv {
+		// Comment is a valid key, its also fully optional
+		// ignore it since it is a valid key for all
+		// sub-systems.
+		if kv.Key == Comment {
+			continue
+		}
+		if _, ok := validKVS.Lookup(kv.Key); !ok {
+			nkv = append(nkv, kv)
+		}
+	}
+	if len(nkv) > 0 {
+		return Errorf(
+			"found invalid keys (%s) for '%s' sub-system, use 'mc admin config reset myminio %s' to fix invalid keys", nkv.String(), subSys, subSys)
+	}
+	return nil
+}
+
+// LookupWorm - check if worm is enabled
+func LookupWorm() (bool, error) {
+	return ParseBool(env.Get(EnvWorm, EnableOff))
+}
+
+// New - initialize a new server config.
+func New() Config {
+	srvCfg := make(Config)
+	for _, k := range SubSystems.ToSlice() {
+		srvCfg[k] = map[string]KVS{}
+		srvCfg[k][Default] = DefaultKVS[k]
+	}
+	return srvCfg
+}
+
+// Target signifies an individual target
+type Target struct {
+	SubSystem string
+	KVS       KVS
+}
+
+// Targets sub-system targets
+type Targets []Target
+
+// GetKVS - get kvs from specific subsystem.
+func (c Config) GetKVS(s string, defaultKVS map[string]KVS) (Targets, error) {
+	if len(s) == 0 {
+		return nil, Errorf("input cannot be empty")
+	}
+	inputs := strings.Fields(s)
+	if len(inputs) > 1 {
+		return nil, Errorf("invalid number of arguments %s", s)
+	}
+	subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
+	if len(subSystemValue) == 0 {
+		return nil, Errorf("invalid number of arguments %s", s)
+	}
+	found := SubSystems.Contains(subSystemValue[0])
+	if !found {
+		// Check for sub-prefix only if the input value is only a
+		// single value, this rejects invalid inputs if any.
+		found = !SubSystems.FuncMatch(strings.HasPrefix, subSystemValue[0]).IsEmpty() && len(subSystemValue) == 1
+	}
+	if !found {
+		return nil, Errorf("unknown sub-system %s", s)
+	}
+
+	targets := Targets{}
+	subSysPrefix := subSystemValue[0]
+	if len(subSystemValue) == 2 {
+		if len(subSystemValue[1]) == 0 {
+			return nil, Errorf("sub-system target '%s' cannot be empty", s)
+		}
+		kvs, ok := c[subSysPrefix][subSystemValue[1]]
+		if !ok {
+			return nil, Errorf("sub-system target '%s' doesn't exist", s)
+		}
+		for _, kv := range defaultKVS[subSysPrefix] {
+			_, ok = kvs.Lookup(kv.Key)
+			if !ok {
+				kvs.Set(kv.Key, kv.Value)
+			}
+		}
+		targets = append(targets, Target{
+			SubSystem: inputs[0],
+			KVS:       kvs,
+		})
+	} else {
+		hkvs := HelpSubSysMap[""]
+		// Use help for sub-system to preserve the order.
+		for _, hkv := range hkvs {
+			if !strings.HasPrefix(hkv.Key, subSysPrefix) {
+				continue
+			}
+			if c[hkv.Key][Default].Empty() {
+				targets = append(targets, Target{
+					SubSystem: hkv.Key,
+					KVS:       defaultKVS[hkv.Key],
+				})
+			}
+			for k, kvs := range c[hkv.Key] {
+				for _, dkv := range defaultKVS[hkv.Key] {
+					_, ok := kvs.Lookup(dkv.Key)
+					if !ok {
+						kvs.Set(dkv.Key, dkv.Value)
+					}
+				}
+				if k != Default {
+					targets = append(targets, Target{
+						SubSystem: hkv.Key + SubSystemSeparator + k,
+						KVS:       kvs,
+					})
+				} else {
+					targets = append(targets, Target{
+						SubSystem: hkv.Key,
+						KVS:       kvs,
+					})
+				}
+			}
+		}
+	}
+	return targets, nil
+}
+
+// DelKVS - delete a specific key.
+func (c Config) DelKVS(s string) error {
+	if len(s) == 0 {
+		return Errorf("input arguments cannot be empty")
+	}
+	inputs := strings.Fields(s)
+	if len(inputs) > 1 {
+		return Errorf("invalid number of arguments %s", s)
+	}
+	subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
+	if len(subSystemValue) == 0 {
+		return Errorf("invalid number of arguments %s", s)
+	}
+	if !SubSystems.Contains(subSystemValue[0]) {
+		// Unknown sub-system found try to remove it anyways.
+		delete(c, subSystemValue[0])
+		return nil
+	}
+	tgt := Default
+	subSys := subSystemValue[0]
+	if len(subSystemValue) == 2 {
+		if len(subSystemValue[1]) == 0 {
+			return Errorf("sub-system target '%s' cannot be empty", s)
+		}
+		tgt = subSystemValue[1]
+	}
+	_, ok := c[subSys][tgt]
+	if !ok {
+		return Errorf("sub-system %s already deleted", s)
+	}
+	delete(c[subSys], tgt)
+	return nil
+}
+
+// Clone - clones a config map entirely.
+func (c Config) Clone() Config {
+	cp := New()
+	for subSys, tgtKV := range c {
+		cp[subSys] = make(map[string]KVS)
+		for tgt, kv := range tgtKV {
+			cp[subSys][tgt] = append(cp[subSys][tgt], kv...)
+		}
+	}
+	return cp
+}
+
+// SetKVS - set specific key values per sub-system.
+func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
+	if len(s) == 0 {
+		return Errorf("input arguments cannot be empty")
+	}
+	inputs := strings.SplitN(s, KvSpaceSeparator, 2)
+	if len(inputs) <= 1 {
+		return Errorf("invalid number of arguments '%s'", s)
+	}
+	subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
+	if len(subSystemValue) == 0 {
+		return Errorf("invalid number of arguments %s", s)
+	}
+
+	if !SubSystems.Contains(subSystemValue[0]) {
+		return Errorf("unknown sub-system %s", s)
+	}
+
+	if SubSystemsSingleTargets.Contains(subSystemValue[0]) && len(subSystemValue) == 2 {
+		return Errorf("sub-system '%s' only supports single target", subSystemValue[0])
+	}
+
+	var kvs = KVS{}
+	var prevK string
+	for _, v := range strings.Fields(inputs[1]) {
+		kv := strings.SplitN(v, KvSeparator, 2)
+		if len(kv) == 0 {
+			continue
+		}
+		if len(kv) == 1 && prevK != "" {
+			value := strings.Join([]string{
+				kvs.Get(prevK),
+				madmin.SanitizeValue(kv[0]),
+			}, KvSpaceSeparator)
+			kvs.Set(prevK, value)
+			continue
+		}
+		if len(kv) == 2 {
+			prevK = kv[0]
+			kvs.Set(prevK, madmin.SanitizeValue(kv[1]))
+			continue
+		}
+		return Errorf("key '%s', cannot have empty value", kv[0])
+	}
+
+	tgt := Default
+	subSys := subSystemValue[0]
+	if len(subSystemValue) == 2 {
+		tgt = subSystemValue[1]
+	}
+
+	_, ok := kvs.Lookup(Enable)
+	// Check if state is required
+	_, enableRequired := defaultKVS[subSys].Lookup(Enable)
+	if !ok && enableRequired {
+		// implicit state "on" if not specified.
+		kvs.Set(Enable, EnableOn)
+	}
+
+	currKVS, ok := c[subSys][tgt]
+	if !ok {
+		currKVS = defaultKVS[subSys]
+	}
+
+	for _, kv := range kvs {
+		if kv.Key == Comment {
+			// Skip comment and add it later.
+			continue
+		}
+		currKVS.Set(kv.Key, kv.Value)
+	}
+
+	v, ok := kvs.Lookup(Comment)
+	if ok {
+		currKVS.Set(Comment, v)
+	}
+
+	hkvs := HelpSubSysMap[subSys]
+	for _, hkv := range hkvs {
+		var enabled bool
+		if enableRequired {
+			enabled = currKVS.Get(Enable) == EnableOn
+		} else {
+			// when enable arg is not required
+			// then it is implicit on for the sub-system.
+			enabled = true
+		}
+		v, _ := currKVS.Lookup(hkv.Key)
+		if v == "" && !hkv.Optional && enabled {
+			// Return error only if the
+			// key is enabled, for state=off
+			// let it be empty.
+			return Errorf(
+				"'%s' is not optional for '%s' sub-system, please check '%s' documentation",
+				hkv.Key, subSys, subSys)
+		}
+	}
+	c[subSys][tgt] = currKVS
+	return nil
+}

cmd/config/config_test.go

@@ -0,0 +1,44 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package config
+
+import "testing"
+
+func TestValidRegion(t *testing.T) {
+	tests := []struct {
+		name    string
+		success bool
+	}{
+		{name: "us-east-1", success: true},
+		{name: "us_east", success: true},
+		{name: "helloWorld", success: true},
+		{name: "-fdslka", success: false},
+		{name: "^00[", success: false},
+		{name: "my region", success: false},
+		{name: "%%$#!", success: false},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ok := validRegionRegex.MatchString(test.name)
+			if test.success != ok {
+				t.Errorf("Expected %t, got %t", test.success, ok)
+			}
+		})
+	}
+}

cmd/config/constants.go

@@ -0,0 +1,40 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package config
+
+// Config value separator
+const (
+	ValueSeparator = ","
+)
+
+// Top level common ENVs
+const (
+	EnvAccessKey    = "MINIO_ACCESS_KEY"
+	EnvSecretKey    = "MINIO_SECRET_KEY"
+	EnvAccessKeyOld = "MINIO_ACCESS_KEY_OLD"
+	EnvSecretKeyOld = "MINIO_SECRET_KEY_OLD"
+	EnvBrowser      = "MINIO_BROWSER"
+	EnvDomain       = "MINIO_DOMAIN"
+	EnvRegionName   = "MINIO_REGION_NAME"
+	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
+	EnvEndpoints    = "MINIO_ENDPOINTS"
+
+	EnvUpdate = "MINIO_UPDATE"
+
+	EnvWorm   = "MINIO_WORM"   // legacy
+	EnvRegion = "MINIO_REGION" // legacy
+)

cmd/config/errors-utils.go

@@ -14,20 +14,22 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
+	"errors"
 	"fmt"
 	"io"
-	"net"
 	"os"
 	"syscall"
+
+	"github.com/minio/minio/pkg/color"
 )
 
-// uiErr is a structure which contains all information
+// Err is a structure which contains all information
 // to print a fatal error message in json or pretty mode
-// uiErr implements error so we can use it anywhere
-type uiErr struct {
+// Err implements error so we can use it anywhere
+type Err struct {
 	msg    string
 	detail string
 	action string
@@ -35,16 +37,19 @@ type uiErr struct {
 }
 
 // Return the error message
-func (u uiErr) Error() string {
+func (u Err) Error() string {
 	if u.detail == "" {
-		return u.msg
+		if u.msg != "" {
+			return u.msg
+		}
+		return "<nil>"
 	}
 	return u.detail
 }
 
-// Replace the current error's message
-func (u uiErr) Msg(m string, args ...interface{}) uiErr {
-	return uiErr{
+// Msg - Replace the current error's message
+func (u Err) Msg(m string, args ...interface{}) Err {
+	return Err{
 		msg:    fmt.Sprintf(m, args...),
 		detail: u.detail,
 		action: u.action,
@@ -52,14 +57,15 @@ func (u uiErr) Msg(m string, args ...interface{}) uiErr {
 	}
 }
 
-type uiErrFn func(err error) uiErr
+// ErrFn function wrapper
+type ErrFn func(err error) Err
 
 // Create a UI error generator, this is needed to simplify
 // the update of the detailed error message in several places
 // in MinIO code
-func newUIErrFn(msg, action, hint string) uiErrFn {
-	return func(err error) uiErr {
-		u := uiErr{
+func newErrFn(msg, action, hint string) ErrFn {
+	return func(err error) Err {
+		u := Err{
 			msg:    msg,
 			action: action,
 			hint:   hint,
@@ -71,47 +77,38 @@ func newUIErrFn(msg, action, hint string) uiErrFn {
 	}
 }
 
-// errorToUIError inspects the passed error and transforms it
+// ErrorToErr inspects the passed error and transforms it
 // to the appropriate UI error.
-func errorToUIErr(err error) uiErr {
-	// If this is already a uiErr, do nothing
-	if e, ok := err.(uiErr); ok {
+func ErrorToErr(err error) Err {
+	if err == nil {
+		return Err{}
+	}
+
+	// If this is already a Err, do nothing
+	if e, ok := err.(Err); ok {
 		return e
 	}
 
 	// Show a generic message for known golang errors
-	switch e := err.(type) {
-	case *net.OpError:
-		if e.Op == "listen" {
-			if oe, ok := e.Err.(*os.SyscallError); ok {
-				if oe.Err == syscall.EADDRINUSE {
-					return uiErrPortAlreadyInUse(e).Msg("Specified port '" + e.Addr.String() + "' is already in use")
-				} else if oe.Err == syscall.EACCES {
-					return uiErrPortAccess(e).Msg("Insufficient permissions to use specified port '" + e.Addr.String() + "'")
-				}
-			}
-		}
-	case *os.PathError:
-		if os.IsPermission(e) {
-			return uiErrNoPermissionsToAccessDirFiles(e).Msg("Insufficient permissions to access path, `" + e.Path + "`")
-		}
-	}
-
-	switch err {
-	case io.ErrUnexpectedEOF:
-		return uiErrUnexpectedDataContent(err)
-	default:
+	if errors.Is(err, syscall.EADDRINUSE) {
+		return ErrPortAlreadyInUse(err).Msg("Specified port is already in use")
+	} else if errors.Is(err, syscall.EACCES) {
+		return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
+	} else if os.IsPermission(err) {
+		return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
+	} else if errors.Is(err, io.ErrUnexpectedEOF) {
+		return ErrUnexpectedDataContent(err)
+	} else {
 		// Failed to identify what type of error this, return a simple UI error
-		return uiErr{msg: err.Error()}
+		return Err{msg: err.Error()}
 	}
-
 }
 
-// fmtError() converts a fatal error message to a more clear error
+// FmtError converts a fatal error message to a more clear error
 // using some colors
-func fmtError(introMsg string, err error, jsonFlag bool) string {
+func FmtError(introMsg string, err error, jsonFlag bool) string {
 	renderedTxt := ""
-	uiErr := errorToUIErr(err)
+	uiErr := ErrorToErr(err)
 	// JSON print
 	if jsonFlag {
 		// Message text in json should be simple
@@ -123,18 +120,18 @@ func fmtError(introMsg string, err error, jsonFlag bool) string {
 	// Pretty print error message
 	introMsg += ": "
 	if uiErr.msg != "" {
-		introMsg += colorBold(uiErr.msg)
+		introMsg += color.Bold(uiErr.msg)
 	} else {
-		introMsg += colorBold(err.Error())
+		introMsg += color.Bold(err.Error())
 	}
-	renderedTxt += colorRed(introMsg) + "\n"
+	renderedTxt += color.Red(introMsg) + "\n"
 	// Add action message
 	if uiErr.action != "" {
-		renderedTxt += "> " + colorBgYellow(colorBlack(uiErr.action)) + "\n"
+		renderedTxt += "> " + color.BgYellow(color.Black(uiErr.action)) + "\n"
 	}
 	// Add hint
 	if uiErr.hint != "" {
-		renderedTxt += colorBold("HINT:") + "\n"
+		renderedTxt += color.Bold("HINT:") + "\n"
 		renderedTxt += "  " + uiErr.hint
 	}
 	return renderedTxt

cmd/config/errors.go

@@ -14,101 +14,113 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
+// UI errors
 var (
-	uiErrInvalidConfig = newUIErrFn(
-		"Invalid value found in the configuration file",
-		"Please ensure a valid value in the configuration file",
-		"For more details, refer to https://docs.min.io/docs/minio-server-configuration-guide",
-	)
-
-	uiErrInvalidBrowserValue = newUIErrFn(
+	ErrInvalidBrowserValue = newErrFn(
 		"Invalid browser value",
 		"Please check the passed value",
 		"Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`",
 	)
 
-	uiErrInvalidDomainValue = newUIErrFn(
+	ErrInvalidDomainValue = newErrFn(
 		"Invalid domain value",
 		"Please check the passed value",
 		"Domain can only accept DNS compatible values",
 	)
 
-	uiErrInvalidErasureSetSize = newUIErrFn(
+	ErrInvalidErasureSetSize = newErrFn(
 		"Invalid erasure set size",
 		"Please check the passed value",
 		"Erasure set can only accept any of [4, 6, 8, 10, 12, 14, 16] values",
 	)
 
-	uiErrInvalidWormValue = newUIErrFn(
+	ErrInvalidWormValue = newErrFn(
 		"Invalid WORM value",
 		"Please check the passed value",
 		"WORM can only accept `on` and `off` values. To enable WORM, set this value to `on`",
 	)
 
-	uiErrInvalidCacheDrivesValue = newUIErrFn(
+	ErrInvalidCacheDrivesValue = newErrFn(
 		"Invalid cache drive value",
 		"Please check the value in this ENV variable",
-		"MINIO_CACHE_DRIVES: Mounted drives or directories are delimited by `;`",
+		"MINIO_CACHE_DRIVES: Mounted drives or directories are delimited by `,`",
 	)
 
-	uiErrInvalidCacheExcludesValue = newUIErrFn(
+	ErrInvalidCacheExcludesValue = newErrFn(
 		"Invalid cache excludes value",
 		"Please check the passed value",
-		"MINIO_CACHE_EXCLUDE: Cache exclusion patterns are delimited by `;`",
+		"MINIO_CACHE_EXCLUDE: Cache exclusion patterns are delimited by `,`",
 	)
 
-	uiErrInvalidCacheExpiryValue = newUIErrFn(
+	ErrInvalidCacheExpiryValue = newErrFn(
 		"Invalid cache expiry value",
 		"Please check the passed value",
-		"MINIO_CACHE_EXPIRY: Valid cache expiry duration is in days",
+		"MINIO_CACHE_EXPIRY: Valid cache expiry duration must be in days",
 	)
 
-	uiErrInvalidCacheMaxUse = newUIErrFn(
-		"Invalid cache max-use value",
+	ErrInvalidCacheQuota = newErrFn(
+		"Invalid cache quota value",
 		"Please check the passed value",
-		"MINIO_CACHE_MAXUSE: Valid cache max-use value between 0-100",
+		"MINIO_CACHE_QUOTA: Valid cache quota value must be between 0-100",
 	)
 
-	uiErrInvalidCacheEncryptionKey = newUIErrFn(
+	ErrInvalidCacheEncryptionKey = newErrFn(
 		"Invalid cache encryption master key value",
 		"Please check the passed value",
 		"MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide",
 	)
 
-	uiErrInvalidCredentials = newUIErrFn(
-		"Invalid credentials",
-		"Please provide correct credentials",
-		`Access key length should be between minimum 3 characters in length.
-Secret key should be in between 8 and 40 characters`,
+	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
+		"Invalid rotating credentials",
+		"Please set correct rotating credentials in the environment for decryption",
+		`Detected encrypted config backend, correct old access and secret keys should be specified via environment variables MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD to be able to re-encrypt the MinIO config, user IAM and policies with new credentials`,
 	)
 
-	uiErrEnvCredentialsMissingGateway = newUIErrFn(
+	ErrInvalidCredentialsBackendEncrypted = newErrFn(
+		"Invalid credentials",
+		"Please set correct credentials in the environment for decryption",
+		`Detected encrypted config backend, correct access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be able to decrypt the MinIO config, user IAM and policies`,
+	)
+
+	ErrMissingCredentialsBackendEncrypted = newErrFn(
+		"Credentials missing",
+		"Please set your credentials in the environment",
+		`Detected encrypted config backend, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be able to decrypt the MinIO config, user IAM and policies`,
+	)
+
+	ErrInvalidCredentials = newErrFn(
+		"Invalid credentials",
+		"Please provide correct credentials",
+		`Access key length should be at least 3, and secret key length at least 8 characters`,
+	)
+
+	ErrEnvCredentialsMissingGateway = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
 		`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
 	)
 
-	uiErrEnvCredentialsMissingDistributed = newUIErrFn(
+	ErrEnvCredentialsMissingDistributed = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
 		`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
 	)
 
-	uiErrInvalidErasureEndpoints = newUIErrFn(
+	ErrInvalidErasureEndpoints = newErrFn(
 		"Invalid endpoint(s) in erasure mode",
 		"Please provide correct combination of local/remote paths",
 		"For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide",
 	)
 
-	uiErrInvalidNumberOfErasureEndpoints = newUIErrFn(
+	ErrInvalidNumberOfErasureEndpoints = newErrFn(
 		"Invalid total number of endpoints for erasure mode",
 		"Please provide an even number of endpoints greater or equal to 4",
 		"For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide",
 	)
 
-	uiErrStorageClassValue = newUIErrFn(
+	ErrStorageClassValue = newErrFn(
 		"Invalid storage class value",
 		"Please check the value",
 		`MINIO_STORAGE_CLASS_STANDARD: Format "EC:<Default_Parity_Standard_Class>" (e.g. "EC:3"). This sets the number of parity disks for MinIO server in Standard mode. Objects are stored in Standard mode, if storage class is not defined in Put request
@@ -116,13 +128,13 @@ MINIO_STORAGE_CLASS_RRS: Format "EC:<Default_Parity_Reduced_Redundancy_Class>" (
 Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storage-class for more information`,
 	)
 
-	uiErrUnexpectedBackendVersion = newUIErrFn(
+	ErrUnexpectedBackendVersion = newErrFn(
 		"Backend version seems to be too recent",
 		"Please update to the latest MinIO version",
 		"",
 	)
 
-	uiErrInvalidAddressFlag = newUIErrFn(
+	ErrInvalidAddressFlag = newErrFn(
 		"--address input is invalid",
 		"Please check --address parameter",
 		`--address binds to a specific ADDRESS:PORT, ADDRESS can be an IPv4/IPv6 address or hostname (default port is ':9000')
@@ -131,7 +143,7 @@ Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storag
 		  --address '[fe80::da00:a6c8:e3ae:ddd7]:9000'`,
 	)
 
-	uiErrInvalidFSEndpoint = newUIErrFn(
+	ErrInvalidFSEndpoint = newErrFn(
 		"Invalid endpoint for standalone FS mode",
 		"Please check the FS endpoint",
 		`FS mode requires only one writable disk path
@@ -139,91 +151,91 @@ Example 1:
    $ minio server /data/minio/`,
 	)
 
-	uiErrUnableToWriteInBackend = newUIErrFn(
+	ErrUnableToWriteInBackend = newErrFn(
 		"Unable to write to the backend",
 		"Please ensure MinIO binary has write permissions for the backend",
 		`Verify if MinIO binary is running as the same user who has write permissions for the backend`,
 	)
 
-	uiErrPortAlreadyInUse = newUIErrFn(
+	ErrPortAlreadyInUse = newErrFn(
 		"Port is already in use",
 		"Please ensure no other program uses the same address/port",
 		"",
 	)
 
-	uiErrPortAccess = newUIErrFn(
+	ErrPortAccess = newErrFn(
 		"Unable to use specified port",
 		"Please ensure MinIO binary has 'cap_net_bind_service=+ep' permissions",
 		`Use 'sudo setcap cap_net_bind_service=+ep /path/to/minio' to provide sufficient permissions`,
 	)
 
-	uiErrNoPermissionsToAccessDirFiles = newUIErrFn(
+	ErrNoPermissionsToAccessDirFiles = newErrFn(
 		"Missing permissions to access the specified path",
 		"Please ensure the specified path can be accessed",
 		"",
 	)
 
-	uiErrSSLUnexpectedError = newUIErrFn(
+	ErrSSLUnexpectedError = newErrFn(
 		"Invalid TLS certificate",
 		"Please check the content of your certificate data",
 		`Only PEM (x.509) format is accepted as valid public & private certificates`,
 	)
 
-	uiErrSSLUnexpectedData = newUIErrFn(
+	ErrSSLUnexpectedData = newErrFn(
 		"Invalid TLS certificate",
 		"Please check your certificate",
 		"",
 	)
 
-	uiErrSSLNoPassword = newUIErrFn(
+	ErrSSLNoPassword = newErrFn(
 		"Missing TLS password",
-		"Please set the password to environment variable `"+TLSPrivateKeyPassword+"` so that the private key can be decrypted",
+		"Please set the password to environment variable `MINIO_CERT_PASSWD` so that the private key can be decrypted",
 		"",
 	)
 
-	uiErrNoCertsAndHTTPSEndpoints = newUIErrFn(
+	ErrNoCertsAndHTTPSEndpoints = newErrFn(
 		"HTTPS specified in endpoints, but no TLS certificate is found on the local machine",
 		"Please add TLS certificate or use HTTP endpoints only",
 		"Refer to https://docs.min.io/docs/how-to-secure-access-to-minio-server-with-tls for information about how to load a TLS certificate in your server",
 	)
 
-	uiErrCertsAndHTTPEndpoints = newUIErrFn(
+	ErrCertsAndHTTPEndpoints = newErrFn(
 		"HTTP specified in endpoints, but the server in the local machine is configured with a TLS certificate",
 		"Please remove the certificate in the configuration directory or switch to HTTPS",
 		"",
 	)
 
-	uiErrSSLWrongPassword = newUIErrFn(
+	ErrSSLWrongPassword = newErrFn(
 		"Unable to decrypt the private key using the provided password",
-		"Please set the correct password in environment variable "+TLSPrivateKeyPassword,
+		"Please set the correct password in environment variable `MINIO_CERT_PASSWD`",
 		"",
 	)
 
-	uiErrUnexpectedDataContent = newUIErrFn(
+	ErrUnexpectedDataContent = newErrFn(
 		"Unexpected data content",
 		"Please contact MinIO at https://slack.min.io",
 		"",
 	)
 
-	uiErrUnexpectedError = newUIErrFn(
+	ErrUnexpectedError = newErrFn(
 		"Unexpected error",
 		"Please contact MinIO at https://slack.min.io",
 		"",
 	)
 
-	uiErrInvalidCompressionIncludesValue = newUIErrFn(
+	ErrInvalidCompressionIncludesValue = newErrFn(
 		"Invalid compression include value",
 		"Please check the passed value",
-		"Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_ATTR=\"A,B,C\"",
+		"Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_MIME_TYPES=\"A,B,C\"",
 	)
 
-	uiErrInvalidGWSSEValue = newUIErrFn(
+	ErrInvalidGWSSEValue = newErrFn(
 		"Invalid gateway SSE value",
 		"Please check the passed value",
 		"MINIO_GATEWAY_SSE: Gateway SSE accepts only C and S3 as valid values. Delimit by `;` to set more than one value",
 	)
 
-	uiErrInvalidGWSSEEnvValue = newUIErrFn(
+	ErrInvalidGWSSEEnvValue = newErrFn(
 		"Invalid gateway SSE configuration",
 		"",
 		"Refer to https://docs.min.io/docs/minio-kms-quickstart-guide.html for setting up SSE",

cmd/config/etcd/dns/etcd_dns.go

@@ -23,14 +23,13 @@ import (
 	"fmt"
 	"net"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/minio/minio-go/v6/pkg/set"
 
 	"github.com/coredns/coredns/plugin/etcd/msg"
-	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/clientv3"
 )
 
 // ErrNoEntriesFound - Indicates no entries were found for the given key (directory)
@@ -39,20 +38,20 @@ var ErrNoEntriesFound = errors.New("No entries found for this key")
 const etcdPathSeparator = "/"
 
 // create a new coredns service record for the bucket.
-func newCoreDNSMsg(bucket, ip string, port int, ttl uint32) ([]byte, error) {
+func newCoreDNSMsg(ip string, port string, ttl uint32) ([]byte, error) {
 	return json.Marshal(&SrvRecord{
 		Host:         ip,
-		Port:         port,
+		Port:         json.Number(port),
 		TTL:          ttl,
 		CreationDate: time.Now().UTC(),
 	})
 }
 
-// Retrieves list of DNS entries for the domain.
-func (c *coreDNS) List() ([]SrvRecord, error) {
+// List - Retrieves list of DNS entries for the domain.
+func (c *CoreDNS) List() ([]SrvRecord, error) {
 	var srvRecords []SrvRecord
 	for _, domainName := range c.domainNames {
-		key := msg.Path(fmt.Sprintf("%s.", domainName), defaultPrefixPath)
+		key := msg.Path(fmt.Sprintf("%s.", domainName), c.prefixPath)
 		records, err := c.list(key)
 		if err != nil {
 			return nil, err
@@ -67,11 +66,11 @@ func (c *coreDNS) List() ([]SrvRecord, error) {
 	return srvRecords, nil
 }
 
-// Retrieves DNS records for a bucket.
-func (c *coreDNS) Get(bucket string) ([]SrvRecord, error) {
+// Get - Retrieves DNS records for a bucket.
+func (c *CoreDNS) Get(bucket string) ([]SrvRecord, error) {
 	var srvRecords []SrvRecord
 	for _, domainName := range c.domainNames {
-		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), defaultPrefixPath)
+		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
 		records, err := c.list(key)
 		if err != nil {
 			return nil, err
@@ -105,9 +104,9 @@ func msgUnPath(s string) string {
 
 // Retrieves list of entries under the key passed.
 // Note that this method fetches entries upto only two levels deep.
-func (c *coreDNS) list(key string) ([]SrvRecord, error) {
+func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-	r, err := c.etcdClient.Get(ctx, key, etcd.WithPrefix())
+	r, err := c.etcdClient.Get(ctx, key, clientv3.WithPrefix())
 	defer cancel()
 	if err != nil {
 		return nil, err
@@ -153,15 +152,17 @@ func (c *coreDNS) list(key string) ([]SrvRecord, error) {
 	return srvRecords, nil
 }
 
-// Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
-func (c *coreDNS) Put(bucket string) error {
+// Put - Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
+func (c *CoreDNS) Put(bucket string) error {
+	c.Delete(bucket) // delete any existing entries.
+
 	for ip := range c.domainIPs {
-		bucketMsg, err := newCoreDNSMsg(bucket, ip, c.domainPort, defaultTTL)
+		bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL)
 		if err != nil {
 			return err
 		}
 		for _, domainName := range c.domainNames {
-			key := msg.Path(fmt.Sprintf("%s.%s", bucket, domainName), defaultPrefixPath)
+			key := msg.Path(fmt.Sprintf("%s.%s", bucket, domainName), c.prefixPath)
 			key = key + etcdPathSeparator + ip
 			ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 			_, err = c.etcdClient.Put(ctx, key, string(bucketMsg))
@@ -177,10 +178,10 @@ func (c *coreDNS) Put(bucket string) error {
 	return nil
 }
 
-// Removes DNS entries added in Put().
-func (c *coreDNS) Delete(bucket string) error {
+// Delete - Removes DNS entries added in Put().
+func (c *CoreDNS) Delete(bucket string) error {
 	for _, domainName := range c.domainNames {
-		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), defaultPrefixPath)
+		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
 		srvRecords, err := c.list(key)
 		if err != nil {
 			return err
@@ -197,10 +198,10 @@ func (c *coreDNS) Delete(bucket string) error {
 	return nil
 }
 
-// Removes a specific DNS entry
-func (c *coreDNS) DeleteRecord(record SrvRecord) error {
+// DeleteRecord - Removes a specific DNS entry
+func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
 	for _, domainName := range c.domainNames {
-		key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), defaultPrefixPath)
+		key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), c.prefixPath)
 
 		dctx, dcancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 		if _, err := c.etcdClient.Delete(dctx, key+etcdPathSeparator+record.Host); err != nil {
@@ -213,26 +214,71 @@ func (c *coreDNS) DeleteRecord(record SrvRecord) error {
 }
 
 // CoreDNS - represents dns config for coredns server.
-type coreDNS struct {
+type CoreDNS struct {
 	domainNames []string
 	domainIPs   set.StringSet
-	domainPort  int
-	etcdClient  *etcd.Client
+	domainPort  string
+	prefixPath  string
+	etcdClient  *clientv3.Client
+}
+
+// Option - functional options pattern style
+type Option func(*CoreDNS)
+
+// DomainNames set a list of domain names used by this CoreDNS
+// client setting, note this will fail if set to empty when
+// constructor initializes.
+func DomainNames(domainNames []string) Option {
+	return func(args *CoreDNS) {
+		args.domainNames = domainNames
+	}
+}
+
+// DomainIPs set a list of custom domain IPs, note this will
+// fail if set to empty when constructor initializes.
+func DomainIPs(domainIPs set.StringSet) Option {
+	return func(args *CoreDNS) {
+		args.domainIPs = domainIPs
+	}
+}
+
+// DomainPort - is a string version of server port
+func DomainPort(domainPort string) Option {
+	return func(args *CoreDNS) {
+		args.domainPort = domainPort
+	}
+}
+
+// CoreDNSPath - custom prefix on etcd to populate DNS
+// service records, optional and can be empty.
+// if empty then c.prefixPath is used i.e "/skydns"
+func CoreDNSPath(prefix string) Option {
+	return func(args *CoreDNS) {
+		args.prefixPath = prefix
+	}
 }
 
 // NewCoreDNS - initialize a new coreDNS set/unset values.
-func NewCoreDNS(domainNames []string, domainIPs set.StringSet, domainPort string, etcdClient *etcd.Client) (Config, error) {
-	if len(domainNames) == 0 || domainIPs.IsEmpty() {
-		return nil, errors.New("invalid argument")
-	}
-
-	port, err := strconv.Atoi(domainPort)
+func NewCoreDNS(cfg clientv3.Config, setters ...Option) (*CoreDNS, error) {
+	etcdClient, err := clientv3.New(cfg)
 	if err != nil {
 		return nil, err
 	}
 
+	args := &CoreDNS{
+		etcdClient: etcdClient,
+	}
+
+	for _, setter := range setters {
+		setter(args)
+	}
+
+	if len(args.domainNames) == 0 || args.domainIPs.IsEmpty() {
+		return nil, errors.New("invalid argument")
+	}
+
 	// strip ports off of domainIPs
-	domainIPsWithoutPorts := domainIPs.ApplyFunc(func(ip string) string {
+	domainIPsWithoutPorts := args.domainIPs.ApplyFunc(func(ip string) string {
 		host, _, err := net.SplitHostPort(ip)
 		if err != nil {
 			if strings.Contains(err.Error(), "missing port in address") {
@@ -241,11 +287,7 @@ func NewCoreDNS(domainNames []string, domainIPs set.StringSet, domainPort string
 		}
 		return host
 	})
+	args.domainIPs = domainIPsWithoutPorts
 
-	return &coreDNS{
-		domainNames: domainNames,
-		domainIPs:   domainIPsWithoutPorts,
-		domainPort:  port,
-		etcdClient:  etcdClient,
-	}, nil
+	return args, nil
 }