Fix logger and audit http endpoint config lookup

- This PR fixes situation to avoid underflow, this is possible
  because of disconnected operations in replay/sendEvents
- Hold right locks if Del() operation is performed in Get()
- Remove panic in the code and use loggerOnce
- Remove Timer and instead use Ticker instead for proper ticks

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.github

.gitignore

@@ -22,4 +22,3 @@ prime/
 stage/
 .sia_temp/
 config.json
-dockerscripts/healthcheck

.travis.yml

@@ -26,25 +26,15 @@ matrix:
         - ARCH=x86_64
         - CGO_ENABLED=0
         - GO111MODULE=on
-        - GOPROXY=https://proxy.golang.org
-      # Enable build cache
-      # https://restic.net/blog/2018-09-02/travis-build-cache
-      cache:
-       directories:
-         - $HOME/.cache/go-build
-         - $HOME/gopath/pkg/mod
-         - $HOME/go/pkg/mod
-
-      go: 1.12.5
+      go: 1.13.x
       script:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
         - diff -au <(gofmt -s -d pkg) <(printf "")
-        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 15m "$d"; done
+        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done
         - make verifiers
         - make crosscompile
         - make verify
-        - make coverage
         - cd browser && yarn && yarn test && cd ..
         - bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'
 
@@ -53,11 +43,10 @@ matrix:
         - ARCH=x86_64
         - CGO_ENABLED=0
         - GO111MODULE=on
-        - GOPROXY=https://proxy.golang.org
-      go: 1.12.5
+      go: 1.13.x
       script:
         - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
-        - bash buildscripts/go-coverage.sh
+        - CGO_ENABLED=1 go test -v --timeout 20m ./...
 
 before_script:
   # Add an IPv6 config - see the corresponding Travis issue
@@ -66,6 +55,3 @@ before_script:
 
 before_install:
  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install 11.10.1 ; fi
-
-after_success:
-- bash <(curl -s https://codecov.io/bash)

Dockerfile

@@ -1,16 +1,16 @@
-FROM golang:1.12-alpine
+FROM golang:1.13-alpine
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 ENV GOPATH /go
 ENV CGO_ENABLED 0
 ENV GO111MODULE on
+ENV GOPROXY https://proxy.golang.org
 
 RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio && cd minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     cd dockerscripts; go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
 
 FROM alpine:3.9
 
@@ -22,7 +22,6 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
-COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
@@ -33,6 +32,4 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=1m CMD healthcheck
-
 CMD ["minio"]

Dockerfile.dev

@@ -2,7 +2,7 @@ FROM alpine:3.9
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck /usr/bin/
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 COPY minio /usr/bin/
 
 ENV MINIO_UPDATE off
@@ -14,8 +14,7 @@ RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      chmod +x /usr/bin/minio  && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck
+     chmod +x /usr/bin/docker-entrypoint.sh
 
 EXPOSE 9000
 
@@ -23,6 +22,4 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=1m CMD healthcheck
-
 CMD ["minio"]

Dockerfile.release

@@ -1,4 +1,4 @@
-FROM golang:1.12-alpine
+FROM golang:1.13-alpine
 
 ENV GOPATH /go
 ENV CGO_ENABLED 0
@@ -6,14 +6,12 @@ ENV GO111MODULE on
 
 RUN  \
      apk add --no-cache git && \
-     git clone https://github.com/minio/minio && cd minio/dockerscripts && \
-     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
+     git clone https://github.com/minio/minio
 
 FROM alpine:3.9
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 ENV MINIO_UPDATE off
@@ -26,8 +24,7 @@ RUN \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      curl https://dl.min.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck
+     chmod +x /usr/bin/docker-entrypoint.sh
 
 EXPOSE 9000
 
@@ -35,6 +32,4 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=1m CMD healthcheck
-
 CMD ["minio"]

Dockerfile.simpleci

@@ -1,14 +1,13 @@
 #-------------------------------------------------------------
 # Stage 1: Build and Unit tests
 #-------------------------------------------------------------
-FROM golang:1.12
+FROM golang:1.13
 
 COPY . /go/src/github.com/minio/minio
 WORKDIR /go/src/github.com/minio/minio
 
 RUN apt-get update && apt-get install -y jq
 ENV GO111MODULE=on
-ENV GOPROXY=https://proxy.golang.org
 
 RUN git config --global http.cookiefile /gitcookie/.gitcookie
 

MAINTAINERS.md

@@ -1,37 +0,0 @@
-# For maintainers only
-
-### Setup your minio GitHub Repository
-
-Fork [minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository.
-```bash
-$ mkdir -p $GOPATH/src/github.com/minio
-$ cd $GOPATH/src/github.com/minio
-$ git clone https://github.com/$USER_ID/minio
-$ 
-```
-
-``minio`` uses [govendor](https://github.com/kardianos/govendor) for its dependency management.
-
-### To manage dependencies
-
-#### Add new dependencies
-
-  - Run `go get foo/bar`
-  - Edit your code to import foo/bar
-  - Run `govendor add foo/bar` from top-level directory
-
-#### Remove dependencies 
-
-  - Run `govendor remove foo/bar`
-
-#### Update dependencies
-
-  - Run `govendor remove +vendor`
-  - Run to update the dependent package `go get -u foo/bar`
-  - Run `govendor add +external`
-
-### Making new releases 
-
-`minio` doesn't follow semantic versioning style, `minio` instead uses the release date and time as the release versions.
-
-`make release` will generate new binary into `release` directory.

Makefile

@@ -2,13 +2,12 @@ PWD := $(shell pwd)
 GOPATH := $(shell go env GOPATH)
 LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 
+GOARCH := $(shell go env GOARCH)
 GOOS := $(shell go env GOOS)
-GOOSALT ?= 'linux'
-ifeq ($(GOOS),'darwin')
-  GOOSALT = 'mac'
-endif
 
-TAG ?= $(USER)
+VERSION ?= $(shell git describe --tags)
+TAG ?= "minio/minio:$(VERSION)"
+
 BUILD_LDFLAGS := '$(LDFLAGS)'
 
 all: build
@@ -19,9 +18,13 @@ checks:
 
 getdeps:
 	@mkdir -p ${GOPATH}/bin
-	@which golint 1>/dev/null || (echo "Installing golint" && go get -u golang.org/x/lint/golint)
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet -O ${GOPATH}/bin/staticcheck https://github.com/dominikh/go-tools/releases/download/2019.1/staticcheck_${GOOS}_amd64 && chmod +x ${GOPATH}/bin/staticcheck)
-	@which misspell 1>/dev/null || (echo "Installing misspell" && wget --quiet https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_${GOOSALT}_64bit.tar.gz && tar xf misspell_0.3.4_${GOOSALT}_64bit.tar.gz && mv misspell ${GOPATH}/bin/misspell && chmod +x ${GOPATH}/bin/misspell && rm -f misspell_0.3.4_${GOOSALT}_64bit.tar.gz)
+	@which golint 1>/dev/null || (echo "Installing golint" && GO111MODULE=off go get -u golang.org/x/lint/golint)
+ifeq ($(GOARCH),s390x)
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && GO111MODULE=off go get honnef.co/go/tools/cmd/staticcheck)
+else
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2019.2.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
+endif
+	@which misspell 1>/dev/null || (echo "Installing misspell" && GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
@@ -29,39 +32,44 @@ crosscompile:
 verifiers: getdeps vet fmt lint staticcheck spelling
 
 vet:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on go vet github.com/minio/minio/...
+	@echo "Running $@ check"
+	@GO111MODULE=on go vet github.com/minio/minio/...
 
 fmt:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on gofmt -d cmd/
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on gofmt -d pkg/
+	@echo "Running $@ check"
+	@GO111MODULE=on gofmt -d cmd/
+	@GO111MODULE=on gofmt -d pkg/
 
 lint:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
+	@echo "Running $@ check"
+	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
+	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
 
 staticcheck:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
+	@echo "Running $@ check"
+	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
+	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
 
 spelling:
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find cmd/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find pkg/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find docs/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
+	@echo "Running $@ check"
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find cmd/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find pkg/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find docs/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build
 	@echo "Running unit tests"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
-verify: build
+# Verify minio binary
+# TODO: enable races as well
+# @GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+verify:
 	@echo "Verifying build"
+	@GO111MODULE=on CGO_ENABLED=1 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 coverage: build
@@ -71,8 +79,7 @@ coverage: build
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

README.md

@@ -5,7 +5,7 @@ MinIO is an object storage server released under Apache License v2.0. It is comp
 
 MinIO server is light enough to be bundled with the application stack, similar to NodeJS, Redis and MySQL.
 
-[1]: MinIO in its default mode is faster and does not calculate MD5Sum unless passed by client. This may lead to incompatibility with few S3 clients like s3ql that heavily depend on MD5Sum. For full compatibility with Amazon S3 API, start MinIO with `--compat` option.
+[1] MinIO in its default mode is faster and does not calculate MD5Sum unless passed by client. This may lead to incompatibility with some S3 clients like s3ql that heavily depend on MD5Sum. For such applications start MinIO with `--compat` option.
 ```sh
 minio --compat server /data
 ```
@@ -91,7 +91,7 @@ service minio start
 Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.12](https://golang.org/dl/#stable)
 
 ```sh
-GOPROXY=https://proxy.golang.org GO111MODULE=on go get github.com/minio/minio
+GO111MODULE=on go get github.com/minio/minio
 ```
 
 ## Allow port access for Firewalls
@@ -162,6 +162,22 @@ When deployed on a single drive, MinIO server lets clients access any pre-existi
 
 The above statement is also valid for all gateway backends.
 
+## Upgrading MinIO
+MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster and restart them, as shown in the following command from the MinIO client (mc):
+
+```
+mc admin update <minio alias, e.g., myminio>
+```
+
+**Important things to remember during upgrades**:
+
+- `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
+- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` until all clusters have been updated.
+- If you are updating the server it is always recommended (unless explicitly mentioned in MinIO server release notes), to update `mc` once all the servers have been upgraded using `mc update`.
+- `mc admin update` is disabled in docker/container environments, container environments provide their own mechanisms for updating running containers.
+- If you are using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
+- If you are using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
+
 ## Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
 - [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)

README_zh_CN.md

@@ -75,7 +75,7 @@ service minio start
 
 ## 使用源码安装
 
-采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://docs.min.io/docs/how-to-install-golang).
+采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install).
 
 ```sh
 go get -u github.com/minio/minio

browser/README.md

@@ -13,7 +13,7 @@ nvm install stable
 
 ### Install `go-bindata` and `go-bindata-assetfs`
 
-If you do not have a working Golang environment, please follow [Install Golang](https://docs.min.io/docs/how-to-install-golang)
+If you do not have a working Golang environment, please follow [Install Golang](https://golang.org/doc/install)
 
 ```sh
 go get github.com/go-bindata/go-bindata/go-bindata

browser/app/js/browser/Login.js

@@ -122,7 +122,6 @@ export class Login extends React.Component {
               type="password"
               spellCheck="false"
               required="required"
-              autoComplete="new-password"
             />
             <button className="lw-btn" type="submit">
               <i className="fas fa-sign-in-alt" />

browser/app/js/buckets/BucketList.js

@@ -17,14 +17,29 @@
 import React from "react"
 import { connect } from "react-redux"
 import { Scrollbars } from "react-custom-scrollbars"
+import InfiniteScroll from "react-infinite-scroller"
 import * as actionsBuckets from "./actions"
-import { getVisibleBuckets } from "./selectors"
+import { getFilteredBuckets } from "./selectors"
 import BucketContainer from "./BucketContainer"
 import web from "../web"
 import history from "../history"
 import { pathSlice } from "../utils"
 
 export class BucketList extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      page: 1
+    }
+    this.loadNextPage = this.loadNextPage.bind(this)
+  }
+  componentDidUpdate(prevProps) {
+    if (this.props.filter !== prevProps.filter) {
+      this.setState({
+        page: 1
+      })
+    }
+  }
   componentWillMount() {
     const { fetchBuckets, setBucketList, selectBucket } = this.props
     if (web.LoggedIn()) {
@@ -39,18 +54,33 @@ export class BucketList extends React.Component {
       }
     }
   }
+  loadNextPage() {
+    this.setState({
+      page: this.state.page + 1
+    })
+  }
   render() {
-    const { visibleBuckets } = this.props
+    const { filteredBuckets } = this.props
+    const visibleBuckets = filteredBuckets.slice(0, this.state.page * 100)
     return (
       <div className="fesl-inner">
         <Scrollbars
           renderTrackVertical={props => <div className="scrollbar-vertical" />}
         >
-          <ul>
-            {visibleBuckets.map(bucket => (
-              <BucketContainer key={bucket} bucket={bucket} />
-            ))}
-          </ul>
+          <InfiniteScroll
+            pageStart={0}
+            loadMore={this.loadNextPage}
+            hasMore={filteredBuckets.length > visibleBuckets.length}
+            useWindow={false}
+            element="div"
+            initialLoad={false}
+          >
+            <ul>
+              {visibleBuckets.map(bucket => (
+                <BucketContainer key={bucket} bucket={bucket} />
+              ))}
+            </ul>
+          </InfiniteScroll>
         </Scrollbars>
       </div>
     )
@@ -59,7 +89,8 @@ export class BucketList extends React.Component {
 
 const mapStateToProps = state => {
   return {
-    visibleBuckets: getVisibleBuckets(state)
+    filteredBuckets: getFilteredBuckets(state),
+    filter: state.buckets.filter
   }
 }
 

browser/app/js/buckets/__tests__/BucketList.test.js

@@ -29,13 +29,13 @@ jest.mock("../../web", () => ({
 describe("BucketList", () => {
   it("should render without crashing", () => {
     const fetchBuckets = jest.fn()
-    shallow(<BucketList visibleBuckets={[]} fetchBuckets={fetchBuckets} />)
+    shallow(<BucketList filteredBuckets={[]} fetchBuckets={fetchBuckets} />)
   })
 
   it("should call fetchBuckets before component is mounted", () => {
     const fetchBuckets = jest.fn()
     const wrapper = shallow(
-      <BucketList visibleBuckets={[]} fetchBuckets={fetchBuckets} />
+      <BucketList filteredBuckets={[]} fetchBuckets={fetchBuckets} />
     )
     expect(fetchBuckets).toHaveBeenCalled()
   })
@@ -46,7 +46,7 @@ describe("BucketList", () => {
     history.push("/bk1/pre1")
     const wrapper = shallow(
       <BucketList
-        visibleBuckets={[]}
+        filteredBuckets={[]}
         setBucketList={setBucketList}
         selectBucket={selectBucket}
       />

browser/app/js/buckets/__tests__/selectors.test.js

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-import { getVisibleBuckets, getCurrentBucket } from "../selectors"
+import { getFilteredBuckets, getCurrentBucket } from "../selectors"
 
-describe("getVisibleBuckets", () => {
+describe("getFilteredBuckets", () => {
   let state
   beforeEach(() => {
     state = {
@@ -28,11 +28,11 @@ describe("getVisibleBuckets", () => {
 
   it("should return all buckets if no filter specified", () => {
     state.buckets.filter = ""
-    expect(getVisibleBuckets(state)).toEqual(["test1", "test11", "test2"])
+    expect(getFilteredBuckets(state)).toEqual(["test1", "test11", "test2"])
   })
 
   it("should return all matching buckets if filter is specified", () => {
     state.buckets.filter = "test1"
-    expect(getVisibleBuckets(state)).toEqual(["test1", "test11"])
+    expect(getFilteredBuckets(state)).toEqual(["test1", "test11"])
   })
 })

browser/app/js/buckets/selectors.js

@@ -19,7 +19,7 @@ import { createSelector } from "reselect"
 const bucketsSelector = state => state.buckets.list
 const bucketsFilterSelector = state => state.buckets.filter
 
-export const getVisibleBuckets = createSelector(
+export const getFilteredBuckets = createSelector(
   bucketsSelector,
   bucketsFilterSelector,
   (buckets, filter) => buckets.filter(bucket => bucket.indexOf(filter) > -1)

browser/package-lock.json

@@ -896,6 +896,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -930,6 +938,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -959,6 +975,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1001,6 +1025,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1073,6 +1105,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1221,6 +1261,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1337,6 +1385,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1540,6 +1596,14 @@
         "babel-runtime": "^6.22.0",
         "core-js": "^2.4.0",
         "regenerator-runtime": "^0.10.0"
+      },
+      "dependencies": {
+        "core-js": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+          "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+          "dev": true
+        }
       }
     },
     "babel-preset-es2015": {
@@ -1653,6 +1717,13 @@
       "requires": {
         "core-js": "^2.4.0",
         "regenerator-runtime": "^0.10.0"
+      },
+      "dependencies": {
+        "core-js": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+          "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A=="
+        }
       }
     },
     "babel-template": {
@@ -1676,6 +1747,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1711,6 +1790,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1741,6 +1828,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1889,9 +1984,9 @@
       "dev": true
     },
     "bootstrap": {
-      "version": "3.3.7",
-      "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.3.7.tgz",
-      "integrity": "sha1-WjiTlFSfIzMIdaOxUGVldPip63E="
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.4.1.tgz",
+      "integrity": "sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA=="
     },
     "brace-expansion": {
       "version": "1.1.11",
@@ -2883,9 +2978,9 @@
       }
     },
     "core-js": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.4.1.tgz",
-      "integrity": "sha1-TekR5mew6ukSTjQlS1OupvxhjT4="
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.2.1.tgz",
+      "integrity": "sha512-Qa5XSVefSVPRxy2XfUC13WbvqkxhkwB3ve+pgCQveNgYzbM/UxZeu1dcOX/xr4UmfUd+muuvsaxilQzCyUurMw=="
     },
     "core-util-is": {
       "version": "1.0.2",
@@ -5326,9 +5421,9 @@
       "dev": true
     },
     "handlebars": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz",
-      "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==",
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.4.0.tgz",
+      "integrity": "sha512-xkRtOt3/3DzTKMOt3xahj2M/EqNhY988T+imYSlMgs5fVhLN2fmKVVj0LtEGmb+3UUYV5Qmm1052Mm3dIQxOvw==",
       "dev": true,
       "requires": {
         "neo-async": "^2.6.0",
@@ -7340,13 +7435,21 @@
       "integrity": "sha1-COnxMkhKLEWjCQfp3E1VZ7fxFNc="
     },
     "js-yaml": {
-      "version": "3.7.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz",
-      "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=",
+      "version": "3.13.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz",
+      "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==",
       "dev": true,
       "requires": {
         "argparse": "^1.0.7",
-        "esprima": "^2.6.0"
+        "esprima": "^4.0.0"
+      },
+      "dependencies": {
+        "esprima": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+          "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+          "dev": true
+        }
       }
     },
     "jsbn": {
@@ -8081,9 +8184,9 @@
       }
     },
     "mixin-deep": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz",
-      "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
+      "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==",
       "requires": {
         "for-in": "^1.0.2",
         "is-extendable": "^1.0.1"
@@ -11648,9 +11751,9 @@
       "dev": true
     },
     "set-value": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
-      "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz",
+      "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==",
       "requires": {
         "extend-shallow": "^2.0.1",
         "is-extendable": "^0.1.1",
@@ -12382,6 +12485,18 @@
         "mkdirp": "~0.5.1",
         "sax": "~1.2.1",
         "whet.extend": "~0.9.9"
+      },
+      "dependencies": {
+        "js-yaml": {
+          "version": "3.7.0",
+          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz",
+          "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=",
+          "dev": true,
+          "requires": {
+            "argparse": "^1.0.7",
+            "esprima": "^2.6.0"
+          }
+        }
       }
     },
     "symbol-observable": {
@@ -12917,35 +13032,14 @@
       }
     },
     "union-value": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
-      "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
+      "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==",
       "requires": {
         "arr-union": "^3.1.0",
         "get-value": "^2.0.6",
         "is-extendable": "^0.1.1",
-        "set-value": "^0.4.3"
-      },
-      "dependencies": {
-        "extend-shallow": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-          "requires": {
-            "is-extendable": "^0.1.0"
-          }
-        },
-        "set-value": {
-          "version": "0.4.3",
-          "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
-          "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
-          "requires": {
-            "extend-shallow": "^2.0.1",
-            "is-extendable": "^0.1.1",
-            "is-plain-object": "^2.0.1",
-            "to-object-path": "^0.3.0"
-          }
-        }
+        "set-value": "^2.0.1"
       }
     },
     "uniq": {

browser/package.json

@@ -60,8 +60,9 @@
   },
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.10.0",
-    "bootstrap": "^3.3.6",
+    "bootstrap": "^3.4.1",
     "classnames": "^2.2.3",
+    "core-js": "^3.2.1",
     "expect": "^1.20.2",
     "glob-all": "^3.1.0",
     "history": "^4.7.2",

buildscripts/checkdeps.sh

@@ -21,7 +21,7 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GO_VERSION="1.12"
+    GO_VERSION="1.13"
     OSX_VERSION="10.8"
     KNAME=$(uname -s)
     ARCH=$(uname -m)
@@ -119,7 +119,7 @@ assert_is_supported_os() {
 
 assert_check_golang_env() {
     if ! which go >/dev/null 2>&1; then
-        echo "Cannot find go binary in your PATH configuration, please refer to Go installation document at https://docs.min.io/docs/how-to-install-golang"
+        echo "Cannot find go binary in your PATH configuration, please refer to Go installation document at https://golang.org/doc/install"
         exit 1
     fi
 

buildscripts/cross-compile.sh

@@ -23,7 +23,6 @@ function _build() {
     export GOOS=$os
     export GOARCH=$arch
     export GO111MODULE=on
-    export GOPROXY=https://proxy.golang.org
     go build -tags kqueue -o /dev/null
 }
 

buildscripts/go-coverage.sh

@@ -2,4 +2,4 @@
 
 set -e
 
-GOPROXY=https://proxy.golang.org GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...
+GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...

buildscripts/verify-build.sh

@@ -33,7 +33,6 @@ export ACCESS_KEY="minio"
 export SECRET_KEY="minio123"
 export ENABLE_HTTPS=0
 export GO111MODULE=on
-export GOPROXY=https://proxy.golang.org
 
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )

cmd/admin-handlers.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018, 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,28 +17,29 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
-	"encoding/base64"
+	"crypto/subtle"
 	"encoding/json"
-	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
+	"path"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
+	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
-	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
 
+	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/cpu"
-	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/handlers"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -50,6 +51,7 @@ import (
 
 const (
 	maxEConfigJSONSize = 262272
+	defaultNetPerfSize = 100 * humanize.MiByte
 )
 
 // Type-safe query params.
@@ -64,107 +66,139 @@ const (
 	mgmtForceStop                = "forceStop"
 )
 
-var (
-	// This struct literal represents the Admin API version that
-	// the server uses.
-	adminAPIVersionInfo = madmin.AdminAPIVersionInfo{
-		Version: "1",
+func updateServer(updateURL, sha256Hex string, latestReleaseTime time.Time) (us madmin.ServerUpdateStatus, err error) {
+	minioMode := getMinioMode()
+	// No inputs provided we should try to update using the default URL.
+	if updateURL == "" && sha256Hex == "" && latestReleaseTime.IsZero() {
+		var updateMsg string
+		updateMsg, sha256Hex, _, latestReleaseTime, err = getUpdateInfo(updateTimeout, minioMode)
+		if err != nil {
+			return us, err
+		}
+		if updateMsg == "" {
+			us.CurrentVersion = Version
+			us.UpdatedVersion = Version
+			return us, nil
+		}
+		if runtime.GOOS == "windows" {
+			updateURL = minioReleaseURL + "minio.exe"
+		} else {
+			updateURL = minioReleaseURL + "minio"
+		}
 	}
-)
+	if err = doUpdate(updateURL, sha256Hex, minioMode); err != nil {
+		return us, err
+	}
+	us.CurrentVersion = Version
+	us.UpdatedVersion = latestReleaseTime.Format(minioReleaseTagTimeLayout)
+	return us, nil
+}
 
-// VersionHandler - GET /minio/admin/version
-// -----------
-// Returns Administration API version
-func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "Version")
+// ServerUpdateHandler - POST /minio/admin/v1/update?updateURL={updateURL}
+// ----------
+// updates all minio servers and restarts them gracefully.
+func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ServerUpdate")
 
 	objectAPI := validateAdminReq(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
 
-	jsonBytes, err := json.Marshal(adminAPIVersionInfo)
+	if globalInplaceUpdateDisabled {
+		// if MINIO_UPDATE=off - inplace update is disabled, mostly
+		// in containers.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	updateURL := vars[peerRESTUpdateURL]
+	mode := getMinioMode()
+	var sha256Hex string
+	var latestReleaseTime time.Time
+	if updateURL != "" {
+		u, err := url.Parse(updateURL)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		content, err := downloadReleaseURL(updateURL, updateTimeout, mode)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		sha256Hex, latestReleaseTime, err = parseReleaseData(content)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		if runtime.GOOS == "windows" {
+			u.Path = path.Dir(u.Path) + "minio.exe"
+		} else {
+			u.Path = path.Dir(u.Path) + "minio"
+		}
+
+		updateURL = u.String()
+	}
+
+	for _, nerr := range globalNotificationSys.ServerUpdate(updateURL, sha256Hex, latestReleaseTime) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	updateStatus, err := updateServer(updateURL, sha256Hex, latestReleaseTime)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	writeSuccessResponseJSON(w, jsonBytes)
-}
-
-// ServiceStatusHandler - GET /minio/admin/v1/service
-// ----------
-// Returns server version and uptime.
-func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ServiceStatus")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	// Fetch server version
-	serverVersion := madmin.ServerVersion{
-		Version:  Version,
-		CommitID: CommitID,
-	}
-
-	// Fetch uptimes from all peers and pick the latest.
-	uptime := getPeerUptimes(globalNotificationSys.ServerInfo(ctx))
-
-	// Create API response
-	serverStatus := madmin.ServiceStatus{
-		ServerVersion: serverVersion,
-		Uptime:        uptime,
-	}
-
 	// Marshal API response
-	jsonBytes, err := json.Marshal(serverStatus)
+	jsonBytes, err := json.Marshal(updateStatus)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Reply with storage information (across nodes in a
-	// distributed setup) as json.
 	writeSuccessResponseJSON(w, jsonBytes)
+
+	if updateStatus.CurrentVersion != updateStatus.UpdatedVersion {
+		// We did upgrade - restart all services.
+		globalServiceSignalCh <- serviceRestart
+	}
 }
 
-// ServiceStopNRestartHandler - POST /minio/admin/v1/service
-// Body: {"action": <restart-action>}
+// ServiceActionHandler - POST /minio/admin/v1/service?action={action}
 // ----------
-// Restarts/Stops minio server gracefully. In a distributed setup,
-// restarts all the servers in the cluster.
-func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ServiceStopNRestart")
+// restarts/stops minio server gracefully. In a distributed setup,
+func (a adminAPIHandlers) ServiceActionHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ServiceAction")
+
+	vars := mux.Vars(r)
+	action := vars["action"]
 
 	objectAPI := validateAdminReq(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
 
-	var sa madmin.ServiceAction
-	err := json.NewDecoder(r.Body).Decode(&sa)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrRequestBodyParse), r.URL)
-		return
-	}
-
 	var serviceSig serviceSignal
-	switch sa.Action {
-	case madmin.ServiceActionValueRestart:
+	switch madmin.ServiceAction(action) {
+	case madmin.ServiceActionRestart:
 		serviceSig = serviceRestart
-	case madmin.ServiceActionValueStop:
+	case madmin.ServiceActionStop:
 		serviceSig = serviceStop
 	default:
+		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action))
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
-		logger.LogIf(ctx, errors.New("Invalid service action received"))
 		return
 	}
 
-	// Reply to the client before restarting minio server.
-	writeSuccessResponseHeadersOnly(w)
-
 	// Notify all other MinIO peers signal service.
 	for _, nerr := range globalNotificationSys.SignalService(serviceSig) {
 		if nerr.Err != nil {
@@ -173,6 +207,9 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 		}
 	}
 
+	// Reply to the client before restarting, stopping MinIO server.
+	writeSuccessResponseHeadersOnly(w)
+
 	globalServiceSignalCh <- serviceSig
 }
 
@@ -275,15 +312,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-// ServerDrivesPerfInfo holds information about address, performance
-// of all drives on one server. It also reports any errors if encountered
-// while trying to reach this server.
-type ServerDrivesPerfInfo struct {
-	Addr  string             `json:"addr"`
-	Error string             `json:"error,omitempty"`
-	Perf  []disk.Performance `json:"perf"`
-}
-
 // ServerCPULoadInfo holds informantion about cpu utilization
 // of one minio node. It also reports any errors if encountered
 // while trying to reach this server.
@@ -304,6 +332,13 @@ type ServerMemUsageInfo struct {
 	HistoricUsage []mem.Usage `json:"historicUsage"`
 }
 
+// ServerNetReadPerfInfo network read performance information.
+type ServerNetReadPerfInfo struct {
+	Addr           string `json:"addr"`
+	ReadThroughput uint64 `json:"readThroughput"`
+	Error          string `json:"error,omitempty"`
+}
+
 // PerfInfoHandler - GET /minio/admin/v1/performance?perfType={perfType}
 // ----------
 // Get all performance information based on input type
@@ -318,17 +353,63 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 
 	vars := mux.Vars(r)
 	switch perfType := vars["perfType"]; perfType {
-	case "drive":
-		info := objectAPI.StorageInfo(ctx)
-		if !(info.Backend.Type == BackendFS || info.Backend.Type == BackendErasure) {
+	case "net":
+		var size int64 = defaultNetPerfSize
+		if sizeStr, found := vars["size"]; found {
+			var err error
+			if size, err = strconv.ParseInt(sizeStr, 10, 64); err != nil || size < 0 {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+				return
+			}
+		}
+
+		if !globalIsDistXL {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 			return
 		}
+
+		addr := r.Host
+		if globalIsDistXL {
+			addr = GetLocalPeer(globalEndpoints)
+		}
+
+		infos := map[string][]ServerNetReadPerfInfo{}
+		infos[addr] = globalNotificationSys.NetReadPerfInfo(size)
+		for peer, info := range globalNotificationSys.CollectNetPerfInfo(size) {
+			infos[peer] = info
+		}
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(infos)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		// Reply with performance information (across nodes in a
+		// distributed setup) as json.
+		writeSuccessResponseJSON(w, jsonBytes)
+
+	case "drive":
+		// Drive Perf is only implemented for Erasure coded backends
+		if !globalIsXL {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+			return
+		}
+
+		var size int64 = madmin.DefaultDrivePerfSize
+		if sizeStr, found := vars["size"]; found {
+			var err error
+			if size, err = strconv.ParseInt(sizeStr, 10, 64); err != nil || size <= 0 {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+				return
+			}
+		}
 		// Get drive performance details from local server's drive(s)
-		dp := localEndpointsDrivePerf(globalEndpoints, r)
+		dp := getLocalDrivesPerf(globalEndpoints, size, r)
 
 		// Notify all other MinIO peers to report drive performance numbers
-		dps := globalNotificationSys.DrivePerfInfo()
+		dps := globalNotificationSys.DrivePerfInfo(size)
 		dps = append(dps, dp)
 
 		// Marshal API response
@@ -343,7 +424,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		writeSuccessResponseJSON(w, jsonBytes)
 	case "cpu":
 		// Get CPU load details from local server's cpu(s)
-		cpu := localEndpointsCPULoad(globalEndpoints, r)
+		cpu := getLocalCPULoad(globalEndpoints, r)
 		// Notify all other MinIO peers to report cpu load numbers
 		cpus := globalNotificationSys.CPULoadInfo()
 		cpus = append(cpus, cpu)
@@ -360,7 +441,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		writeSuccessResponseJSON(w, jsonBytes)
 	case "mem":
 		// Get mem usage details from local server(s)
-		m := localEndpointsMemUsage(globalEndpoints, r)
+		m := getLocalMemUsage(globalEndpoints, r)
 		// Notify all other MinIO peers to report mem usage numbers
 		mems := globalNotificationSys.MemUsageInfo()
 		mems = append(mems, m)
@@ -391,7 +472,6 @@ func newLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntr
 }
 
 func topLockEntries(peerLocks []*PeerLocks) madmin.LockEntries {
-	const listCount int = 10
 	entryMap := make(map[string]*madmin.LockEntry)
 	for _, peerLock := range peerLocks {
 		if peerLock == nil {
@@ -412,6 +492,7 @@ func topLockEntries(peerLocks []*PeerLocks) madmin.LockEntries {
 		lockEntries = append(lockEntries, *v)
 	}
 	sort.Sort(lockEntries)
+	const listCount int = 10
 	if len(lockEntries) > listCount {
 		lockEntries = lockEntries[:listCount]
 	}
@@ -569,45 +650,60 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
 	}
 }
 
+type healInitParams struct {
+	bucket, objPrefix     string
+	hs                    madmin.HealOpts
+	clientToken           string
+	forceStart, forceStop bool
+}
+
 // extractHealInitParams - Validates params for heal init API.
-func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
-	hs madmin.HealOpts, clientToken string, forceStart bool, forceStop bool,
-	err APIErrorCode) {
+func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reader) (hip healInitParams, err APIErrorCode) {
+	hip.bucket = vars[string(mgmtBucket)]
+	hip.objPrefix = vars[string(mgmtPrefix)]
 
-	vars := mux.Vars(r)
-	bucket = vars[string(mgmtBucket)]
-	objPrefix = vars[string(mgmtPrefix)]
-
-	if bucket == "" {
-		if objPrefix != "" {
+	if hip.bucket == "" {
+		if hip.objPrefix != "" {
 			// Bucket is required if object-prefix is given
 			err = ErrHealMissingBucket
 			return
 		}
-	} else if isReservedOrInvalidBucket(bucket, false) {
+	} else if isReservedOrInvalidBucket(hip.bucket, false) {
 		err = ErrInvalidBucketName
 		return
 	}
 
 	// empty prefix is valid.
-	if !IsValidObjectPrefix(objPrefix) {
+	if !IsValidObjectPrefix(hip.objPrefix) {
 		err = ErrInvalidObjectName
 		return
 	}
 
-	qParms := r.URL.Query()
 	if len(qParms[string(mgmtClientToken)]) > 0 {
-		clientToken = qParms[string(mgmtClientToken)][0]
+		hip.clientToken = qParms[string(mgmtClientToken)][0]
 	}
 	if _, ok := qParms[string(mgmtForceStart)]; ok {
-		forceStart = true
+		hip.forceStart = true
 	}
 	if _, ok := qParms[string(mgmtForceStop)]; ok {
-		forceStop = true
+		hip.forceStop = true
 	}
+
+	// Invalid request conditions:
+	//
+	//   Cannot have both forceStart and forceStop in the same
+	//   request; If clientToken is provided, request can only be
+	//   to continue receiving logs, so it cannot be start or
+	//   stop;
+	if (hip.forceStart && hip.forceStop) ||
+		(hip.clientToken != "" && (hip.forceStart || hip.forceStop)) {
+		err = ErrInvalidRequest
+		return
+	}
+
 	// ignore body if clientToken is provided
-	if clientToken == "" {
-		jerr := json.NewDecoder(r.Body).Decode(&hs)
+	if hip.clientToken == "" {
+		jerr := json.NewDecoder(r).Decode(&hip.hs)
 		if jerr != nil {
 			logger.LogIf(context.Background(), jerr)
 			err = ErrRequestBodyParse
@@ -646,7 +742,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	bucket, objPrefix, hs, clientToken, forceStart, forceStop, errCode := extractHealInitParams(r)
+	hip, errCode := extractHealInitParams(mux.Vars(r), r.URL.Query(), r.Body)
 	if errCode != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
 		return
@@ -720,8 +816,8 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	info := objectAPI.StorageInfo(ctx)
 	numDisks := info.Backend.OfflineDisks + info.Backend.OnlineDisks
 
-	healPath := pathJoin(bucket, objPrefix)
-	if clientToken == "" && !forceStart && !forceStop {
+	healPath := pathJoin(hip.bucket, hip.objPrefix)
+	if hip.clientToken == "" && !hip.forceStart && !hip.forceStop {
 		nh, exists := globalAllHealState.getHealSequence(healPath)
 		if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
 			b, err := json.Marshal(madmin.HealStartSuccess{
@@ -740,11 +836,11 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	if clientToken != "" && !forceStart && !forceStop {
+	if hip.clientToken != "" && !hip.forceStart && !hip.forceStop {
 		// Since clientToken is given, fetch heal status from running
 		// heal sequence.
 		respBytes, errCode := globalAllHealState.PopHealStatusJSON(
-			healPath, clientToken)
+			healPath, hip.clientToken)
 		if errCode != ErrNone {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
 		} else {
@@ -755,14 +851,14 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 
 	respCh := make(chan healResp)
 	switch {
-	case forceStop:
+	case hip.forceStop:
 		go func() {
 			respBytes, apiErr := globalAllHealState.stopHealSequence(healPath)
 			hr := healResp{respBytes: respBytes, apiErr: apiErr}
 			respCh <- hr
 		}()
-	case clientToken == "":
-		nh := newHealSequence(bucket, objPrefix, handlers.GetSourceIP(r), numDisks, hs, forceStart)
+	case hip.clientToken == "":
+		nh := newHealSequence(hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), numDisks, hip.hs, hip.forceStart)
 		go func() {
 			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
 			hr := healResp{respBytes, apiErr, errMsg}
@@ -851,25 +947,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	writeSuccessResponseJSON(w, econfigData)
 }
 
-// Disable tidwall json array notation in JSON key path so
-// users can set json with a key as a number.
-// In tidwall json, notify.webhook.0 = val means { "notify" : { "webhook" : [val] }}
-// In MinIO, notify.webhook.0 = val means { "notify" : { "webhook" : {"0" : val}}}
-func normalizeJSONKey(input string) (key string) {
-	subKeys := strings.Split(input, ".")
-	for i, k := range subKeys {
-		if i > 0 {
-			key += "."
-		}
-		if _, err := strconv.Atoi(k); err == nil {
-			key += ":" + k
-		} else {
-			key += k
-		}
-	}
-	return
-}
-
 func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -888,60 +965,24 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 	return objectAPI
 }
 
-// GetConfigHandler - GET /minio/admin/v1/config-keys
-// Get some keys in config.json of this minio setup.
-func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetConfigKeysHandler")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	var keys []string
-	queries := r.URL.Query()
-
-	for k := range queries {
-		keys = append(keys, k)
-	}
-
-	config, err := readServerConfig(ctx, objectAPI)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	configData, err := json.Marshal(config)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	configStr := string(configData)
-	newConfigStr := `{}`
-
-	for _, key := range keys {
-		// sjson.Set does not return an error if key is empty
-		// we should check by ourselves here
-		if key == "" {
-			continue
-		}
-		val := gjson.Get(configStr, key)
-		if j, ierr := sjson.Set(newConfigStr, normalizeJSONKey(key), val.Value()); ierr == nil {
-			newConfigStr = j
-		}
-	}
-
-	password := config.GetCredential().SecretKey
-	econfigData, err := madmin.EncryptData(password, []byte(newConfigStr))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	writeSuccessResponseJSON(w, []byte(econfigData))
+// AdminError - is a generic error for all admin APIs.
+type AdminError struct {
+	Code       string
+	Message    string
+	StatusCode int
 }
 
+func (ae AdminError) Error() string {
+	return ae.Message
+}
+
+// Admin API errors
+const (
+	AdminUpdateUnexpectedFailure = "XMinioAdminUpdateUnexpectedFailure"
+	AdminUpdateURLNotReachable   = "XMinioAdminUpdateURLNotReachable"
+	AdminUpdateApplyFailure      = "XMinioAdminUpdateApplyFailure"
+)
+
 // toAdminAPIErrCode - converts errXLWriteQuorum error to admin API
 // specific error.
 func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
@@ -954,7 +995,21 @@ func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
 }
 
 func toAdminAPIErr(ctx context.Context, err error) APIError {
-	return errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
+	if err == nil {
+		return noError
+	}
+	apiErr := errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
+	if apiErr.Code == "InternalError" {
+		switch e := err.(type) {
+		case AdminError:
+			apiErr = APIError{
+				Code:           e.Code,
+				Description:    e.Message,
+				HTTPStatusCode: e.StatusCode,
+			}
+		}
+	}
+	return apiErr
 }
 
 // RemoveUser - DELETE /minio/admin/v1/remove-user?accessKey=<access_key>
@@ -1126,7 +1181,12 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	groups := globalIAMSys.ListGroups()
+	groups, err := globalIAMSys.ListGroups()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	body, err := json.Marshal(groups)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1269,6 +1329,25 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// InfoCannedPolicy - GET /minio/admin/v1/info-canned-policy?name={policyName}
+func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "InfoCannedPolicy")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	data, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.Write(data)
+	w.(http.Flusher).Flush()
+}
+
 // ListCannedPolicies - GET /minio/admin/v1/list-canned-policies
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPolicies")
@@ -1486,127 +1565,6 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	writeSuccessResponseHeadersOnly(w)
 }
 
-func convertValueType(elem []byte, jsonType gjson.Type) (interface{}, error) {
-	str := string(elem)
-	switch jsonType {
-	case gjson.False, gjson.True:
-		return strconv.ParseBool(str)
-	case gjson.JSON:
-		return gjson.Parse(str).Value(), nil
-	case gjson.String:
-		return str, nil
-	case gjson.Number:
-		return strconv.ParseFloat(str, 64)
-	default:
-		return nil, nil
-	}
-}
-
-// SetConfigKeysHandler - PUT /minio/admin/v1/config-keys
-func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetConfigKeysHandler")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	// Load config
-	configStruct, err := readServerConfig(ctx, objectAPI)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Convert config to json bytes
-	configBytes, err := json.Marshal(configStruct)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	configStr := string(configBytes)
-
-	queries := r.URL.Query()
-	password := globalServerConfig.GetCredential().SecretKey
-
-	// Set key values in the JSON config
-	for k := range queries {
-		// Decode encrypted data associated to the current key
-		encryptedElem, dErr := base64.StdEncoding.DecodeString(queries.Get(k))
-		if dErr != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("key", k)
-			ctx = logger.SetReqInfo(ctx, reqInfo)
-			logger.LogIf(ctx, dErr)
-			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), dErr.Error(), r.URL)
-			return
-		}
-
-		elem, dErr := madmin.DecryptData(password, bytes.NewBuffer([]byte(encryptedElem)))
-		if dErr != nil {
-			logger.LogIf(ctx, dErr)
-			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), dErr.Error(), r.URL)
-			return
-		}
-
-		// Calculate the type of the current key from the
-		// original config json
-		jsonFieldType := gjson.Get(configStr, k).Type
-		// Convert passed value to json filed type
-		val, cErr := convertValueType(elem, jsonFieldType)
-		if cErr != nil {
-			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), cErr.Error(), r.URL)
-			return
-		}
-		// Set the key/value in the new json document
-		if s, sErr := sjson.Set(configStr, normalizeJSONKey(k), val); sErr == nil {
-			configStr = s
-		}
-	}
-
-	configBytes = []byte(configStr)
-
-	// Validate config
-	var config serverConfig
-	if err = json.Unmarshal(configBytes, &config); err != nil {
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
-		return
-	}
-
-	if err = config.Validate(); err != nil {
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
-		return
-	}
-
-	if err = config.TestNotificationTargets(); err != nil {
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
-		return
-	}
-
-	// If credentials for the server are provided via environment,
-	// then credentials in the provided configuration must match.
-	if globalIsEnvCreds {
-		if !globalServerConfig.GetCredential().Equal(config.Credential) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminCredentialsMismatch), r.URL)
-			return
-		}
-	}
-
-	if err = saveServerConfig(ctx, objectAPI, &config); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Send success response
-	writeSuccessResponseHeadersOnly(w)
-}
-
 // Returns true if the trace.Info should be traced,
 // false if certain conditions are not met.
 // - input entry is not of the type *trace.Info*
@@ -1682,3 +1640,189 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 }
+
+// The handler sends console logs to the connected HTTP client.
+func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ConsoleLog")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+	node := r.URL.Query().Get("node")
+	// limit buffered console entries if client requested it.
+	limitStr := r.URL.Query().Get("limit")
+	limitLines, err := strconv.Atoi(limitStr)
+	if err != nil {
+		limitLines = 10
+	}
+	// Avoid reusing tcp connection if read timeout is hit
+	// This is needed to make r.Context().Done() work as
+	// expected in case of read timeout
+	w.Header().Add("Connection", "close")
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+	logCh := make(chan interface{}, 4000)
+
+	remoteHosts := getRemoteHosts(globalEndpoints)
+	peers, err := getRestClients(remoteHosts)
+	if err != nil {
+		return
+	}
+
+	globalConsoleSys.Subscribe(logCh, doneCh, node, limitLines, nil)
+
+	for _, peer := range peers {
+		if node == "" || strings.EqualFold(peer.host.Name, node) {
+			peer.ConsoleLog(logCh, doneCh)
+		}
+	}
+
+	enc := json.NewEncoder(w)
+
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+
+	for {
+		select {
+		case entry := <-logCh:
+			log := entry.(madmin.LogInfo)
+			if log.SendLog(node) {
+				if err := enc.Encode(log); err != nil {
+					return
+				}
+				w.(http.Flusher).Flush()
+			}
+		case <-keepAliveTicker.C:
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-GlobalServiceDoneCh:
+			return
+		}
+	}
+}
+
+// KMSKeyStatusHandler - GET /minio/admin/v1/kms/key/status?key-id=<master-key-id>
+func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "KMSKeyStatusHandler")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	if GlobalKMS == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
+		return
+	}
+
+	keyID := r.URL.Query().Get("key-id")
+	if keyID == "" {
+		keyID = GlobalKMS.KeyID()
+	}
+	var response = madmin.KMSKeyStatus{
+		KeyID: keyID,
+	}
+
+	kmsContext := crypto.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation
+	// 1. Generate a new key using the KMS.
+	key, sealedKey, err := GlobalKMS.GenerateKey(keyID, kmsContext)
+	if err != nil {
+		response.EncryptionErr = err.Error()
+		resp, err := json.Marshal(response)
+		if err != nil {
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+			return
+		}
+		writeSuccessResponseJSON(w, resp)
+		return
+	}
+
+	// 2. Check whether we can update / re-wrap the sealed key.
+	sealedKey, err = GlobalKMS.UpdateKey(keyID, sealedKey, kmsContext)
+	if err != nil {
+		response.UpdateErr = err.Error()
+		resp, err := json.Marshal(response)
+		if err != nil {
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+			return
+		}
+		writeSuccessResponseJSON(w, resp)
+		return
+	}
+
+	// 3. Verify that we can indeed decrypt the (encrypted) key
+	decryptedKey, err := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
+	if err != nil {
+		response.DecryptionErr = err.Error()
+		resp, err := json.Marshal(response)
+		if err != nil {
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+			return
+		}
+		writeSuccessResponseJSON(w, resp)
+		return
+	}
+
+	// 4. Compare generated key with decrypted key
+	if subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1 {
+		response.DecryptionErr = "The generated and the decrypted data key do not match"
+		resp, err := json.Marshal(response)
+		if err != nil {
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+			return
+		}
+		writeSuccessResponseJSON(w, resp)
+		return
+	}
+
+	resp, err := json.Marshal(response)
+	if err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+		return
+	}
+	writeSuccessResponseJSON(w, resp)
+}
+
+// ServerHardwareInfoHandler - GET /minio/admin/v1/hardwareinfo?Type={hwType}
+// ----------
+// Get all hardware information based on input type
+// Supported types = cpu
+func (a adminAPIHandlers) ServerHardwareInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "HardwareInfo")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	hardware := vars[madmin.HARDWARE]
+
+	switch madmin.HardwareType(hardware) {
+	case madmin.CPU:
+		// Get CPU hardware details from local server's cpu(s)
+		cpu := getLocalCPUInfo(globalEndpoints, r)
+		// Notify all other MinIO peers to report cpu hardware
+		cpus := globalNotificationSys.CPUInfo()
+		cpus = append(cpus, cpu)
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(cpus)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		// Reply with cpu hardware information (across nodes in a
+		// distributed setup) as json.
+		writeSuccessResponseJSON(w, jsonBytes)
+
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+	}
+}

cmd/admin-handlers_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,6 +28,7 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/pkg/auth"
@@ -293,11 +294,16 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	globalIAMSys = NewIAMSys()
 	globalIAMSys.Init(objLayer)
 
+	buckets, err := objLayer.ListBuckets(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
 	globalPolicySys = NewPolicySys()
-	globalPolicySys.Init(objLayer)
+	globalPolicySys.Init(buckets, objLayer)
 
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
-	globalNotificationSys.Init(objLayer)
+	globalNotificationSys.Init(buckets, objLayer)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
@@ -344,117 +350,35 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) {
 	return objLayer, xlDirs, nil
 }
 
-func TestAdminVersionHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	req, err := newTestRequest("GET", "/minio/admin/version", 0, nil)
-	if err != nil {
-		t.Fatalf("Failed to construct request - %v", err)
-	}
-	cred := globalServerConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign request - %v", err)
-	}
-
-	rec := httptest.NewRecorder()
-	adminTestBed.router.ServeHTTP(rec, req)
-	if http.StatusOK != rec.Code {
-		t.Errorf("Unexpected status code - got %d but expected %d",
-			rec.Code, http.StatusOK)
-	}
-
-	var result madmin.AdminAPIVersionInfo
-	err = json.NewDecoder(rec.Body).Decode(&result)
-	if err != nil {
-		t.Errorf("json parse err: %v", err)
-	}
-
-	if result != adminAPIVersionInfo {
-		t.Errorf("unexpected version: %v", result)
-	}
-}
-
 // cmdType - Represents different service subcomands like status, stop
 // and restart.
 type cmdType int
 
 const (
-	statusCmd cmdType = iota
-	restartCmd
+	restartCmd cmdType = iota
 	stopCmd
-	setCreds
 )
 
-// String - String representation for cmdType
-func (c cmdType) String() string {
-	switch c {
-	case statusCmd:
-		return "status"
-	case restartCmd:
-		return "restart"
-	case stopCmd:
-		return "stop"
-	case setCreds:
-		return "set-credentials"
-	}
-	return ""
-}
-
-// apiMethod - Returns the HTTP method corresponding to the admin REST
-// API for a given cmdType value.
-func (c cmdType) apiMethod() string {
-	switch c {
-	case statusCmd:
-		return "GET"
-	case restartCmd:
-		return "POST"
-	case stopCmd:
-		return "POST"
-	case setCreds:
-		return "PUT"
-	}
-	return "GET"
-}
-
-// apiEndpoint - Return endpoint for each admin REST API mapped to a
-// command here.
-func (c cmdType) apiEndpoint() string {
-	switch c {
-	case statusCmd, restartCmd, stopCmd:
-		return "/minio/admin/v1/service"
-	case setCreds:
-		return "/minio/admin/v1/config/credential"
-	}
-	return ""
-}
-
 // toServiceSignal - Helper function that translates a given cmdType
 // value to its corresponding serviceSignal value.
 func (c cmdType) toServiceSignal() serviceSignal {
 	switch c {
-	case statusCmd:
-		return serviceStatus
 	case restartCmd:
 		return serviceRestart
 	case stopCmd:
 		return serviceStop
 	}
-	return serviceStatus
+	return serviceRestart
 }
 
-func (c cmdType) toServiceActionValue() madmin.ServiceActionValue {
+func (c cmdType) toServiceAction() madmin.ServiceAction {
 	switch c {
 	case restartCmd:
-		return madmin.ServiceActionValueRestart
+		return madmin.ServiceActionRestart
 	case stopCmd:
-		return madmin.ServiceActionValueStop
+		return madmin.ServiceActionStop
 	}
-	return madmin.ServiceActionValueStop
+	return madmin.ServiceActionRestart
 }
 
 // testServiceSignalReceiver - Helper function that simulates a
@@ -469,19 +393,15 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 
 // getServiceCmdRequest - Constructs a management REST API request for service
 // subcommands for a given cmdType value.
-func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*http.Request, error) {
-	req, err := newTestRequest(cmd.apiMethod(), cmd.apiEndpoint(), 0, nil)
+func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, error) {
+	queryVal := url.Values{}
+	queryVal.Set("action", string(cmd.toServiceAction()))
+	resource := "/minio/admin/v1/service?" + queryVal.Encode()
+	req, err := newTestRequest(http.MethodPost, resource, 0, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	// Set body
-	req.Body = ioutil.NopCloser(bytes.NewReader(body))
-	req.ContentLength = int64(len(body))
-
-	// Set sha-sum header
-	req.Header.Set("X-Amz-Content-Sha256", getSHA256Hash(body))
-
 	// management REST API uses signature V4 for authentication.
 	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 	if err != nil {
@@ -517,13 +437,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	}
 	credentials := globalServerConfig.GetCredential()
 
-	body, err := json.Marshal(madmin.ServiceAction{
-		Action: cmd.toServiceActionValue()})
-	if err != nil {
-		t.Fatalf("JSONify error: %v", err)
-	}
-
-	req, err := getServiceCmdRequest(cmd, credentials, body)
+	req, err := getServiceCmdRequest(cmd, credentials)
 	if err != nil {
 		t.Fatalf("Failed to build service status request %v", err)
 	}
@@ -531,19 +445,6 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	rec := httptest.NewRecorder()
 	adminTestBed.router.ServeHTTP(rec, req)
 
-	if cmd == statusCmd {
-		expectedInfo := madmin.ServiceStatus{
-			ServerVersion: madmin.ServerVersion{Version: Version, CommitID: CommitID},
-		}
-		receivedInfo := madmin.ServiceStatus{}
-		if jsonErr := json.Unmarshal(rec.Body.Bytes(), &receivedInfo); jsonErr != nil {
-			t.Errorf("Failed to unmarshal StorageInfo - %v", jsonErr)
-		}
-		if expectedInfo.ServerVersion != receivedInfo.ServerVersion {
-			t.Errorf("Expected storage info and received storage info differ, %v %v", expectedInfo, receivedInfo)
-		}
-	}
-
 	if rec.Code != http.StatusOK {
 		resp, _ := ioutil.ReadAll(rec.Body)
 		t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
@@ -554,11 +455,6 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	wg.Wait()
 }
 
-// Test for service status management REST API.
-func TestServiceStatusHandler(t *testing.T) {
-	testServicesCmdHandler(statusCmd, t)
-}
-
 // Test for service restart management REST API.
 func TestServiceRestartHandler(t *testing.T) {
 	testServicesCmdHandler(restartCmd, t)
@@ -760,3 +656,100 @@ func TestToAdminAPIErrCode(t *testing.T) {
 		}
 	}
 }
+
+func TestTopLockEntries(t *testing.T) {
+	t1 := UTCNow()
+	t2 := UTCNow().Add(10 * time.Second)
+	peerLocks := []*PeerLocks{
+		{
+			Addr: "1",
+			Locks: map[string][]lockRequesterInfo{
+				"1": {
+					{false, "node2", "ep2", "2", t2, t2, ""},
+					{true, "node1", "ep1", "1", t1, t1, ""},
+				},
+				"2": {
+					{false, "node2", "ep2", "2", t2, t2, ""},
+					{true, "node1", "ep1", "1", t1, t1, ""},
+				},
+			},
+		},
+		{
+			Addr: "2",
+			Locks: map[string][]lockRequesterInfo{
+				"1": {
+					{false, "node2", "ep2", "2", t2, t2, ""},
+					{true, "node1", "ep1", "1", t1, t1, ""},
+				},
+				"2": {
+					{false, "node2", "ep2", "2", t2, t2, ""},
+					{true, "node1", "ep1", "1", t1, t1, ""},
+				},
+			},
+		},
+	}
+	les := topLockEntries(peerLocks)
+	if len(les) != 2 {
+		t.Fatalf("Did not get 2 results")
+	}
+	if les[0].Timestamp.After(les[1].Timestamp) {
+		t.Fatalf("Got wrong sorted value")
+	}
+}
+
+func TestExtractHealInitParams(t *testing.T) {
+	mkParams := func(clientToken string, forceStart, forceStop bool) url.Values {
+		v := url.Values{}
+		if clientToken != "" {
+			v.Add(string(mgmtClientToken), clientToken)
+		}
+		if forceStart {
+			v.Add(string(mgmtForceStart), "")
+		}
+		if forceStop {
+			v.Add(string(mgmtForceStop), "")
+		}
+		return v
+	}
+	qParmsArr := []url.Values{
+		// Invalid cases
+		mkParams("", true, true),
+		mkParams("111", true, true),
+		mkParams("111", true, false),
+		mkParams("111", false, true),
+		// Valid cases follow
+		mkParams("", true, false),
+		mkParams("", false, true),
+		mkParams("", false, false),
+		mkParams("111", false, false),
+	}
+	varsArr := []map[string]string{
+		// Invalid cases
+		{string(mgmtPrefix): "objprefix"},
+		// Valid cases
+		{},
+		{string(mgmtBucket): "bucket"},
+		{string(mgmtBucket): "bucket", string(mgmtPrefix): "objprefix"},
+	}
+
+	// Body is always valid - we do not test JSON decoding.
+	body := `{"recursive": false, "dryRun": true, "remove": false, "scanMode": 0}`
+
+	// Test all combinations!
+	for pIdx, parms := range qParmsArr {
+		for vIdx, vars := range varsArr {
+			_, err := extractHealInitParams(vars, parms, bytes.NewBuffer([]byte(body)))
+			isErrCase := false
+			if pIdx < 4 || vIdx < 1 {
+				isErrCase = true
+			}
+
+			if err != ErrNone && !isErrCase {
+				t.Errorf("Got unexpected error: %v %v %v", pIdx, vIdx, err)
+			} else if err == ErrNone && isErrCase {
+				t.Errorf("Got no error but expected one: %v %v", pIdx, vIdx)
+			}
+		}
+	}
+
+}

cmd/admin-router.go

@@ -38,20 +38,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 	adminRouter := router.PathPrefix(adminAPIPathPrefix).Subrouter()
 
 	// Version handler
-	adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(httpTraceAll(adminAPI.VersionHandler))
-
 	adminV1Router := adminRouter.PathPrefix("/v1").Subrouter()
 
 	/// Service operations
 
-	// Service status
-	adminV1Router.Methods(http.MethodGet).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceStatusHandler))
-
-	// Service restart and stop - TODO
-	adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceStopNRestartHandler))
+	// Restart and stop MinIO service.
+	adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
+	// Update MinIO servers.
+	adminV1Router.Methods(http.MethodPost).Path("/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
 
 	// Info operations
 	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
+	// Harware Info operations
+	adminV1Router.Methods(http.MethodGet).Path("/hardware").HandlerFunc(httpTraceAll(adminAPI.ServerHardwareInfoHandler)).Queries("hwType", "{hwType:.*}")
 
 	if globalIsDistXL || globalIsXL {
 		/// Heal operations
@@ -80,11 +79,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 		// Set config
 		adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
-
-		// Get config keys/values
-		adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
-		// Set config keys/values
-		adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
 	}
 
 	if enableIAMOps {
@@ -99,6 +93,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
 			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
+		// Info policy IAM
+		adminV1Router.Methods(http.MethodGet).Path("/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
+
 		// Remove policy IAM
 		adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
 
@@ -138,6 +135,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 	// HTTP Trace
 	adminV1Router.Methods(http.MethodGet).Path("/trace").HandlerFunc(adminAPI.TraceHandler)
+
+	// Console Logs
+	adminV1Router.Methods(http.MethodGet).Path("/log").HandlerFunc(httpTraceAll(adminAPI.ConsoleLogHandler))
+
+	// -- KMS APIs --
+	//
+	adminV1Router.Methods(http.MethodGet).Path("/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
+
 	// If none of the routes match, return error.
-	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
+	adminV1Router.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(versionMismatchHandler))
 }

cmd/admin-server-info.go

@@ -0,0 +1,149 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/http"
+	"os"
+
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/cpu"
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/mem"
+
+	cpuhw "github.com/shirou/gopsutil/cpu"
+)
+
+// getLocalMemUsage - returns ServerMemUsageInfo for only the
+// local endpoints from given list of endpoints
+func getLocalMemUsage(endpoints EndpointList, r *http.Request) ServerMemUsageInfo {
+	var memUsages []mem.Usage
+	var historicUsages []mem.Usage
+	seenHosts := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if seenHosts.Contains(endpoint.Host) {
+			continue
+		}
+		seenHosts.Add(endpoint.Host)
+
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			memUsages = append(memUsages, mem.GetUsage())
+			historicUsages = append(historicUsages, mem.GetHistoricUsage())
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
+	return ServerMemUsageInfo{
+		Addr:          addr,
+		Usage:         memUsages,
+		HistoricUsage: historicUsages,
+	}
+}
+
+// getLocalCPULoad - returns ServerCPULoadInfo for only the
+// local endpoints from given list of endpoints
+func getLocalCPULoad(endpoints EndpointList, r *http.Request) ServerCPULoadInfo {
+	var cpuLoads []cpu.Load
+	var historicLoads []cpu.Load
+	seenHosts := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if seenHosts.Contains(endpoint.Host) {
+			continue
+		}
+		seenHosts.Add(endpoint.Host)
+
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			cpuLoads = append(cpuLoads, cpu.GetLoad())
+			historicLoads = append(historicLoads, cpu.GetHistoricLoad())
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
+	return ServerCPULoadInfo{
+		Addr:         addr,
+		Load:         cpuLoads,
+		HistoricLoad: historicLoads,
+	}
+}
+
+// getLocalDrivesPerf - returns ServerDrivesPerfInfo for only the
+// local endpoints from given list of endpoints
+func getLocalDrivesPerf(endpoints EndpointList, size int64, r *http.Request) madmin.ServerDrivesPerfInfo {
+	var dps []disk.Performance
+	for _, endpoint := range endpoints {
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			if _, err := os.Stat(endpoint.Path); err != nil {
+				// Since this drive is not available, add relevant details and proceed
+				dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
+				continue
+			}
+			dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()), size)
+			dp.Path = endpoint.Path
+			dps = append(dps, dp)
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
+	return madmin.ServerDrivesPerfInfo{
+		Addr: addr,
+		Perf: dps,
+		Size: size,
+	}
+}
+
+// getLocalCPUInfo - returns ServerCPUHardwareInfo only for the
+// local endpoints from given list of endpoints
+func getLocalCPUInfo(endpoints EndpointList, r *http.Request) madmin.ServerCPUHardwareInfo {
+	var cpuHardwares []cpuhw.InfoStat
+	seenHosts := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if seenHosts.Contains(endpoint.Host) {
+			continue
+		}
+		// Add to the list of visited hosts
+		seenHosts.Add(endpoint.Host)
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			cpuHardware, err := cpuhw.Info()
+			if err != nil {
+				return madmin.ServerCPUHardwareInfo{
+					Error: err.Error(),
+				}
+			}
+			cpuHardwares = append(cpuHardwares, cpuHardware...)
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
+
+	return madmin.ServerCPUHardwareInfo{
+		Addr:    addr,
+		CPUInfo: cpuHardwares,
+	}
+}

cmd/api-errors.go

@@ -141,6 +141,7 @@ const (
 	ErrInvalidPrefixMarker
 	ErrBadRequest
 	ErrKeyTooLongError
+	ErrInvalidAPIVersion
 	// Add new error codes here.
 
 	// SSE-S3 related API errors
@@ -315,6 +316,7 @@ const (
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
+	ErrPostPolicyConditionInvalidFormat
 )
 
 type errorCodeMap map[APIErrorCode]APIError
@@ -1496,6 +1498,16 @@ var errorCodes = errorCodeMap{
 		Description:    "User is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrPostPolicyConditionInvalidFormat: {
+		Code:           "PostPolicyInvalidKeyName",
+		Description:    "Invalid according to Policy: Policy Condition failed",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrInvalidAPIVersion: {
+		Code:           "ErrInvalidAPIVersion",
+		Description:    "Invalid version found in the request",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	// Add your error structure here.
 }
 
@@ -1526,6 +1538,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooLarge
 	case errDataTooSmall:
 		apiErr = ErrEntityTooSmall
+	case errAuthentication:
+		apiErr = ErrAccessDenied
 	case auth.ErrInvalidAccessKeyLength:
 		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidSecretKeyLength:
@@ -1721,6 +1735,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// their internal error types. This code is only
 		// useful with gateway implementations.
 		switch e := err.(type) {
+		case crypto.Error:
+			apiErr = APIError{
+				Code:           "XKMSInternalError",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case minio.ErrorResponse:
 			apiErr = APIError{
 				Code:           e.Code,

cmd/api-errors_test.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"context"
 	"errors"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/minio/minio/cmd/crypto"
@@ -65,6 +67,11 @@ var toAPIErrorTests = []struct {
 }
 
 func TestAPIErrCode(t *testing.T) {
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(disk)
+
+	initFSObjects(disk, t)
+
 	ctx := context.Background()
 	for i, testCase := range toAPIErrorTests {
 		errCode := toAPIErrorCode(ctx, testCase.err)

cmd/api-resources.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"net/url"
 	"strconv"
 )
@@ -42,6 +43,27 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
 	return
 }
 
+func getListBucketObjectVersionsArgs(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType, versionIDMarker string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	if values.Get("max-keys") != "" {
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
+	} else {
+		maxkeys = maxObjectList
+	}
+
+	prefix = values.Get("prefix")
+	marker = values.Get("key-marker")
+	delimiter = values.Get("delimiter")
+	encodingType = values.Get("encoding-type")
+	versionIDMarker = values.Get("version-id-marker")
+	return
+}
+
 // Parse bucket url queries for ListObjects V2.
 func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string, errCode APIErrorCode) {
 	errCode = ErrNone
@@ -65,11 +87,19 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 	}
 
 	prefix = values.Get("prefix")
-	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
+
+	if token = values.Get("continuation-token"); token != "" {
+		decodedToken, err := base64.StdEncoding.DecodeString(token)
+		if err != nil {
+			errCode = ErrIncorrectContinuationToken
+			return
+		}
+		token = string(decodedToken)
+	}
 	return
 }
 

cmd/api-resources_test.go

@@ -34,7 +34,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		{
 			values: url.Values{
 				"prefix":             []string{"photos/"},
-				"continuation-token": []string{"token"},
+				"continuation-token": []string{"dG9rZW4="},
 				"start-after":        []string{"start-after"},
 				"delimiter":          []string{SlashSeparator},
 				"fetch-owner":        []string{"true"},
@@ -53,7 +53,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		{
 			values: url.Values{
 				"prefix":             []string{"photos/"},
-				"continuation-token": []string{"token"},
+				"continuation-token": []string{"dG9rZW4="},
 				"start-after":        []string{"start-after"},
 				"delimiter":          []string{SlashSeparator},
 				"fetch-owner":        []string{"true"},

cmd/api-response.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/xml"
 	"net/http"
 	"net/url"
@@ -44,6 +45,45 @@ type LocationResponse struct {
 	Location string   `xml:",chardata"`
 }
 
+// ListVersionsResponse - format for list bucket versions response.
+type ListVersionsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult" json:"-"`
+
+	Name      string
+	Prefix    string
+	KeyMarker string
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects. Server lists objects in alphabetical
+	// order Note: This element is returned only if you have delimiter request parameter
+	// specified. If response does not include the NextMaker and it is truncated,
+	// you can use the value of the last Key in the response as the marker in the
+	// subsequent request to get the next set of object keys.
+	NextKeyMarker string `xml:"NextKeyMarker,omitempty"`
+
+	// When the number of responses exceeds the value of MaxKeys,
+	// NextVersionIdMarker specifies the first object version not
+	// returned that satisfies the search criteria. Use this value
+	// for the version-id-marker request parameter in a subsequent request.
+	NextVersionIDMarker string `xml:"NextVersionIdMarker"`
+
+	// Marks the last version of the Key returned in a truncated response.
+	VersionIDMarker string `xml:"VersionIdMarker"`
+
+	MaxKeys   int
+	Delimiter string
+	// A flag that indicates whether or not ListObjects returned all of the results
+	// that satisfied the search criteria.
+	IsTruncated bool
+
+	CommonPrefixes []CommonPrefix
+	Versions       []ObjectVersion
+
+	// Encoding type used to encode object keys in the response.
+	EncodingType string `xml:"EncodingType,omitempty"`
+}
+
 // ListObjectsResponse - format for list objects response.
 type ListObjectsResponse struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
@@ -191,6 +231,14 @@ type Bucket struct {
 	CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
 }
 
+// ObjectVersion container for object version metadata
+type ObjectVersion struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Version" json:"-"`
+	Object
+	VersionID string `xml:"VersionId"`
+	IsLatest  bool
+}
+
 // Object container for object metadata
 type Object struct {
 	Key          string
@@ -328,6 +376,52 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	return data
 }
 
+// generates an ListBucketVersions response for the said bucket with other enumerated options.
+func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListVersionsResponse {
+	var versions []ObjectVersion
+	var prefixes []CommonPrefix
+	var owner = Owner{}
+	var data = ListVersionsResponse{}
+
+	owner.ID = globalMinioDefaultOwnerID
+	for _, object := range resp.Objects {
+		var content = ObjectVersion{}
+		if object.Name == "" {
+			continue
+		}
+		content.Key = s3EncodeName(object.Name, encodingType)
+		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
+		if object.ETag != "" {
+			content.ETag = "\"" + object.ETag + "\""
+		}
+		content.Size = object.Size
+		content.StorageClass = object.StorageClass
+		content.Owner = owner
+		content.VersionID = "null"
+		content.IsLatest = true
+		versions = append(versions, content)
+	}
+	data.Name = bucket
+	data.Versions = versions
+
+	data.EncodingType = encodingType
+	data.Prefix = s3EncodeName(prefix, encodingType)
+	data.KeyMarker = s3EncodeName(marker, encodingType)
+	data.Delimiter = s3EncodeName(delimiter, encodingType)
+	data.MaxKeys = maxKeys
+
+	data.NextKeyMarker = s3EncodeName(resp.NextMarker, encodingType)
+	data.IsTruncated = resp.IsTruncated
+
+	for _, prefix := range resp.Prefixes {
+		var prefixItem = CommonPrefix{}
+		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
+		prefixes = append(prefixes, prefixItem)
+	}
+	data.CommonPrefixes = prefixes
+	return data
+}
+
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
 func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
 	var contents []Object
@@ -405,8 +499,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	data.Delimiter = s3EncodeName(delimiter, encodingType)
 	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.MaxKeys = maxKeys
-	data.ContinuationToken = token
-	data.NextContinuationToken = nextToken
+	data.ContinuationToken = base64.StdEncoding.EncodeToString([]byte(token))
+	data.NextContinuationToken = base64.StdEncoding.EncodeToString([]byte(nextToken))
 	data.IsTruncated = isTruncated
 	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
@@ -609,6 +703,17 @@ func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIE
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
 
+// writeVersionMismatchResponse - writes custom error responses for version mismatches.
+func writeVersionMismatchResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL, isJSON bool) {
+	if isJSON {
+		// Generate error response.
+		errorResponse := getAPIErrorResponse(ctx, err, reqURL.String(), w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+		writeResponse(w, err.HTTPStatusCode, encodeResponseJSON(errorResponse), mimeJSON)
+	} else {
+		writeResponse(w, err.HTTPStatusCode, []byte(err.Description), mimeNone)
+	}
+}
+
 // writeCustomErrorResponseJSON - similar to writeErrorResponseJSON,
 // but accepts the error message directly (this allows messages to be
 // dynamically generated.)

cmd/api-router.go

@@ -64,7 +64,7 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-		// ListObjectPxarts
+		// ListObjectParts
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
@@ -129,6 +129,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
 		// ListObjectsV2
 		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
+		// ListBucketVersions
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListBucketObjectVersionsHandler)).Queries("versions", "")
 		// ListObjectsV1 (Legacy)
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
 		// PutBucketLifecycle

cmd/auth-handler.go

@@ -203,6 +203,12 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 	}
 
 	if globalPolicyOPA == nil {
+		// If OPA is not set and if ldap claim key is set,
+		// allow the claim.
+		if _, ok := claims[ldapUser]; ok {
+			return claims, nil
+		}
+
 		// If OPA is not set, session token should
 		// have a policy and its mandatory, reject
 		// requests without policy claim.

cmd/background-heal-ops.go

@@ -62,11 +62,11 @@ func (h *healRoutine) run() {
 				break
 			}
 			if globalHTTPServer != nil {
-				// Wait at max 1 minute for an inprogress request
-				// before proceeding to heal
-				waitCount := 60
+				// Wait at max 10 minute for an inprogress request before proceeding to heal
+				waitCount := 600
 				// Any requests in progress, delay the heal.
-				for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+				for (globalHTTPServer.GetRequestCount() >= int32(globalXLSetCount*globalXLSetDriveCount)) &&
+					waitCount > 0 {
 					waitCount--
 					time.Sleep(1 * time.Second)
 				}

cmd/bitrot-streaming.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/hex"
+	"fmt"
 	"hash"
 	"io"
 
@@ -131,7 +132,8 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err = HashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
+		err = fmt.Errorf("hashes do not match expected %s, got %s",
+			hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))
 		logger.LogIf(context.Background(), err)
 		return 0, err
 	}

cmd/bitrot.go

@@ -155,3 +155,11 @@ func bitrotWriterSum(w io.Writer) []byte {
 	}
 	return nil
 }
+
+// Returns the size of the file with bitrot protection
+func bitrotShardFileSize(size int64, shardSize int64, algo BitrotAlgorithm) int64 {
+	if algo != HighwayHash256S {
+		return size
+	}
+	return ceilFrac(size, shardSize)*int64(algo.New().Size()) + size
+}

cmd/bucket-handlers-listobjects.go

@@ -49,6 +49,83 @@ func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, max
 	return ErrNone
 }
 
+// ListBucketObjectVersions - GET Bucket Object versions
+// You can use the versions subresource to list metadata about all
+// of the versions of objects in a bucket.
+func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListBucketObjectVersions")
+
+	defer logger.AuditLog(w, r, "ListBucketObjectVersions", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	urlValues := r.URL.Query()
+
+	// Extract all the listBucketVersions query params to their native values.
+	// versionIDMarker is ignored here.
+	prefix, marker, delimiter, maxkeys, encodingType, _, errCode := getListBucketObjectVersionsArgs(urlValues)
+	if errCode != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Validate the query params before beginning to serve the request.
+	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxkeys); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	listObjects := objectAPI.ListObjects
+
+	// Inititate a list objects operation based on the input params.
+	// On success would return back ListObjectsInfo object to be
+	// marshaled into S3 compatible XML header.
+	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxkeys)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	for i := range listObjectsInfo.Objects {
+		var actualSize int64
+		if listObjectsInfo.Objects[i].IsCompressed() {
+			// Read the decompressed size from the meta.json.
+			actualSize = listObjectsInfo.Objects[i].GetActualSize()
+			if actualSize < 0 {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize),
+					r.URL, guessIsBrowserReq(r))
+				return
+			}
+			// Set the info.Size to the actualSize.
+			listObjectsInfo.Objects[i].Size = actualSize
+		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
+			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
+			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+				return
+			}
+		}
+	}
+
+	response := generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingType, maxkeys, listObjectsInfo)
+
+	// Write success response.
+	writeSuccessResponseXML(w, encodeResponse(response))
+}
+
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 1000)

cmd/bucket-handlers.go

@@ -288,6 +288,9 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	// Set delimiter value for "s3:delimiter" policy conditionals.
 	r.Header.Set("delimiter", SlashSeparator)
 
+	// err will be nil here as we already called this function
+	// earlier in this request.
+	claims, _ := getClaimsFromToken(r)
 	var newBucketsInfo []BucketInfo
 	for _, bucketInfo := range bucketsInfo {
 		if globalIAMSys.IsAllowed(iampolicy.Args{
@@ -297,6 +300,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 			ConditionValues: getConditionValues(r, "", accessKey),
 			IsOwner:         owner,
 			ObjectName:      "",
+			Claims:          claims,
 		}) {
 			newBucketsInfo = append(newBucketsInfo, bucketInfo)
 		}
@@ -553,8 +557,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
-	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -653,9 +656,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Handle policy if it is set.
 	if len(policyBytes) > 0 {
+
 		postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
 		if err != nil {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPostPolicyConditionInvalidFormat), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -711,7 +715,11 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, SlashSeparator) { // handle SSE-C and SSE-S3 requests
+		if crypto.IsRequested(formValues) && !hasSuffix(object, SlashSeparator) { // handle SSE requests
+			if crypto.SSECopy.IsRequested(r.Header) {
+				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
+				return
+			}
 			var reader io.Reader
 			var key []byte
 			if crypto.SSEC.IsRequested(formValues) {

cmd/common-main.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,25 +17,23 @@
 package cmd
 
 import (
-	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"net"
-	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"time"
 
-	etcd "github.com/coreos/etcd/clientv3"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/logger/target/console"
-	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/dns"
-	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/env"
 )
 
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
@@ -46,7 +44,7 @@ func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
 
 	if strings.HasPrefix(name, "gateway") {
 		if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
-			uiErr := uiErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
+			uiErr := config.ErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
 			logger.Fatal(uiErr, "Unable to start gateway with SSE")
 		}
 	}
@@ -65,43 +63,13 @@ func checkUpdate(mode string) {
 			return
 		}
 		if globalInplaceUpdateDisabled {
-			logger.StartupMessage(updateMsg)
+			logStartupMessage(updateMsg)
 		} else {
-			logger.StartupMessage(prepareUpdateMessage("Run `minio update`", latestReleaseTime.Sub(currentReleaseTime)))
+			logStartupMessage(prepareUpdateMessage("Run `mc admin update`", latestReleaseTime.Sub(currentReleaseTime)))
 		}
 	}
 }
 
-// Load logger targets based on user's configuration
-func loadLoggers() {
-	loggerUserAgent := getUserAgent(getMinioMode())
-
-	auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
-	if ok {
-		// Enable audit HTTP logging through ENV.
-		logger.AddAuditTarget(http.New(auditEndpoint, loggerUserAgent, NewCustomHTTPTransport()))
-	}
-
-	loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
-	if ok {
-		// Enable HTTP logging through ENV.
-		logger.AddTarget(http.New(loggerEndpoint, loggerUserAgent, NewCustomHTTPTransport()))
-	} else {
-		for _, l := range globalServerConfig.Logger.HTTP {
-			if l.Enabled {
-				// Enable http logging
-				logger.AddTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
-			}
-		}
-	}
-
-	if globalServerConfig.Logger.Console.Enabled {
-		// Enable console logging
-		logger.AddTarget(console.New())
-	}
-
-}
-
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
 	var dir string
 	var dirSet bool
@@ -190,31 +158,13 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	globalCLIContext.StrictS3Compat = ctx.IsSet("compat") || ctx.GlobalIsSet("compat")
 }
 
-// Parses the given compression exclude list `extensions` or `content-types`.
-func parseCompressIncludes(includes []string) ([]string, error) {
-	for _, e := range includes {
-		if len(e) == 0 {
-			return nil, uiErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
-		}
-	}
-	return includes, nil
-}
-
 func handleCommonEnvVars() {
-	compressEnvDelimiter := ","
-	// Start profiler if env is set.
-	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
-		var err error
-		globalProfiler, err = startProfiler(profiler, "")
-		logger.FatalIf(err, "Unable to setup a profiler")
-	}
-
-	accessKey := os.Getenv("MINIO_ACCESS_KEY")
-	secretKey := os.Getenv("MINIO_SECRET_KEY")
+	accessKey := env.Get(config.EnvAccessKey, "")
+	secretKey := env.Get(config.EnvSecretKey, "")
 	if accessKey != "" && secretKey != "" {
 		cred, err := auth.CreateCredentials(accessKey, secretKey)
 		if err != nil {
-			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
+			logger.Fatal(config.ErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
 		}
 		cred.Expiration = timeSentinel
 
@@ -223,10 +173,10 @@ func handleCommonEnvVars() {
 		globalActiveCred = cred
 	}
 
-	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
-		browserFlag, err := ParseBoolFlag(browser)
+	if browser := env.Get(config.EnvBrowser, "on"); browser != "" {
+		browserFlag, err := config.ParseBoolFlag(browser)
 		if err != nil {
-			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
+			logger.Fatal(config.ErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -235,65 +185,23 @@ func handleCommonEnvVars() {
 		globalIsBrowserEnabled = bool(browserFlag)
 	}
 
-	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
-	if ok {
-		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
-
-		var etcdSecure bool
-		for _, endpoint := range etcdEndpoints {
-			u, err := xnet.ParseURL(endpoint)
-			if err != nil {
-				logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
-			}
-			// If one of the endpoint is https, we will use https directly.
-			etcdSecure = etcdSecure || u.Scheme == "https"
-		}
-
-		var err error
-		if etcdSecure {
-			// This is only to support client side certificate authentication
-			// https://coreos.com/etcd/docs/latest/op-guide/security.html
-			etcdClientCertFile, ok1 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT")
-			etcdClientCertKey, ok2 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT_KEY")
-			var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
-			if ok1 && ok2 {
-				getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-					cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
-					return &cert, terr
-				}
-			}
-
-			globalEtcdClient, err = etcd.New(etcd.Config{
-				Endpoints:         etcdEndpoints,
-				DialTimeout:       defaultDialTimeout,
-				DialKeepAliveTime: defaultDialKeepAlive,
-				TLS: &tls.Config{
-					RootCAs:              globalRootCAs,
-					GetClientCertificate: getClientCertificate,
-				},
-			})
-		} else {
-			globalEtcdClient, err = etcd.New(etcd.Config{
-				Endpoints:         etcdEndpoints,
-				DialTimeout:       defaultDialTimeout,
-				DialKeepAliveTime: defaultDialKeepAlive,
-			})
-		}
-		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+	var err error
+	globalEtcdClient, err = etcd.New(globalRootCAs)
+	if err != nil {
+		logger.FatalIf(err, "Unable to initialize etcd config")
 	}
 
-	v, ok := os.LookupEnv("MINIO_DOMAIN")
-	if ok {
-		for _, domainName := range strings.Split(v, ",") {
-			if _, ok = dns2.IsDomainName(domainName); !ok {
-				logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
+	for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), ",") {
+		if domainName != "" {
+			if _, ok := dns2.IsDomainName(domainName); !ok {
+				logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
 					"Invalid MINIO_DOMAIN value in environment variable")
 			}
 			globalDomainNames = append(globalDomainNames, domainName)
 		}
 	}
 
-	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	minioEndpointsEnv, ok := env.Lookup(config.EnvPublicIPs)
 	if ok {
 		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
 		var domainIPs = set.NewStringSet()
@@ -324,81 +232,16 @@ func handleCommonEnvVars() {
 		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainNames)
 	}
 
-	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
-		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
-		if err != nil {
-			logger.Fatal(err, "Unable to parse MINIO_CACHE_DRIVES value (%s)", drives)
-		}
-		globalCacheDrives = driveList
-		globalIsDiskCacheEnabled = true
-	}
-
-	if excludes := os.Getenv("MINIO_CACHE_EXCLUDE"); excludes != "" {
-		excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter))
-		if err != nil {
-			logger.Fatal(err, "Unable to parse MINIO_CACHE_EXCLUDE value (`%s`)", excludes)
-		}
-		globalCacheExcludes = excludeList
-	}
-
-	if expiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); expiryStr != "" {
-		expiry, err := strconv.Atoi(expiryStr)
-		if err != nil {
-			logger.Fatal(uiErrInvalidCacheExpiryValue(err), "Unable to parse MINIO_CACHE_EXPIRY value (`%s`)", expiryStr)
-		}
-		globalCacheExpiry = expiry
-	}
-
-	if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" {
-		maxUse, err := strconv.Atoi(maxUseStr)
-		if err != nil {
-			logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr)
-		}
-		// maxUse should be a valid percentage.
-		if maxUse > 0 && maxUse <= 100 {
-			globalCacheMaxUse = maxUse
-		}
-	}
 	// In place update is true by default if the MINIO_UPDATE is not set
 	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
 	// in-place update is off.
-	globalInplaceUpdateDisabled = strings.EqualFold(os.Getenv("MINIO_UPDATE"), "off")
-
-	// Validate and store the storage class env variables only for XL/Dist XL setups
-	if globalIsXL {
-		var err error
-
-		// Check for environment variables and parse into storageClass struct
-		if ssc := os.Getenv(standardStorageClassEnv); ssc != "" {
-			globalStandardStorageClass, err = parseStorageClass(ssc)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
-		}
-
-		if rrsc := os.Getenv(reducedRedundancyStorageClassEnv); rrsc != "" {
-			globalRRStorageClass, err = parseStorageClass(rrsc)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
-		}
-
-		// Validation is done after parsing both the storage classes. This is needed because we need one
-		// storage class value to deduce the correct value of the other storage class.
-		if globalRRStorageClass.Scheme != "" {
-			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
-			globalIsStorageClass = true
-		}
-
-		if globalStandardStorageClass.Scheme != "" {
-			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
-			globalIsStorageClass = true
-		}
-	}
+	globalInplaceUpdateDisabled = strings.EqualFold(env.Get(config.EnvUpdate, "off"), "off")
 
 	// Get WORM environment variable.
-	if worm := os.Getenv("MINIO_WORM"); worm != "" {
-		wormFlag, err := ParseBoolFlag(worm)
+	if worm := env.Get(config.EnvWorm, "off"); worm != "" {
+		wormFlag, err := config.ParseBoolFlag(worm)
 		if err != nil {
-			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
+			logger.Fatal(config.ErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
 		}
 
 		// worm Envs are set globally, this does not represent
@@ -407,27 +250,29 @@ func handleCommonEnvVars() {
 		globalWORMEnabled = bool(wormFlag)
 	}
 
-	if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
-		globalIsCompressionEnabled = strings.EqualFold(compress, "true")
+}
+
+func logStartupMessage(msg string, data ...interface{}) {
+	if globalConsoleSys != nil {
+		globalConsoleSys.Send(msg)
+	}
+	logger.StartupMessage(msg, data...)
+}
+
+func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
+	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
+		return nil, nil, false, nil
 	}
 
-	compressExtensions := os.Getenv("MINIO_COMPRESS_EXTENSIONS")
-	compressMimeTypes := os.Getenv("MINIO_COMPRESS_MIMETYPES")
-	if compressExtensions != "" || compressMimeTypes != "" {
-		globalIsEnvCompression = true
-		if compressExtensions != "" {
-			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter))
-			if err != nil {
-				logger.Fatal(err, "Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", extensions)
-			}
-			globalCompressExtensions = extensions
-		}
-		if compressMimeTypes != "" {
-			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter))
-			if err != nil {
-				logger.Fatal(err, "Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", contenttypes)
-			}
-			globalCompressMimeTypes = contenttypes
-		}
+	if x509Certs, err = config.ParsePublicCertFile(getPublicCertFile()); err != nil {
+		return nil, nil, false, err
 	}
+
+	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), config.LoadX509KeyPair)
+	if err != nil {
+		return nil, nil, false, err
+	}
+
+	secureConn = true
+	return x509Certs, c, secureConn, nil
 }

cmd/config-current.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,19 +20,25 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
-	"reflect"
+	"strings"
 	"sync"
 
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/cache"
+	"github.com/minio/minio/cmd/config/compress"
+	xldap "github.com/minio/minio/cmd/config/ldap"
+	"github.com/minio/minio/cmd/config/notify"
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/openid"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/iam/validator"
-	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Steps to move from version N to version N+1
@@ -78,6 +84,10 @@ func (s *serverConfig) GetRegion() string {
 
 // SetCredential sets new credential and returns the previous credential.
 func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
+	if s == nil {
+		return creds
+	}
+
 	if creds.IsValid() && globalActiveCred.IsValid() {
 		globalActiveCred = creds
 	}
@@ -103,24 +113,16 @@ func (s *serverConfig) GetCredential() auth.Credentials {
 // SetWorm set if worm is enabled.
 func (s *serverConfig) SetWorm(b bool) {
 	// Set the new value.
-	s.Worm = BoolFlag(b)
-}
-
-func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
-	s.StorageClass.Standard = standardClass
-	s.StorageClass.RRS = rrsClass
+	s.Worm = config.BoolFlag(b)
 }
 
 // GetStorageClass reads storage class fields from current config.
 // It returns the standard and reduced redundancy storage class struct
-func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
-	if globalIsStorageClass {
-		return globalStandardStorageClass, globalRRStorageClass
-	}
+func (s *serverConfig) GetStorageClass() storageclass.Config {
 	if s == nil {
-		return storageClass{}, storageClass{}
+		return storageclass.Config{}
 	}
-	return s.StorageClass.Standard, s.StorageClass.RRS
+	return s.StorageClass
 }
 
 // GetWorm get current credentials.
@@ -134,18 +136,10 @@ func (s *serverConfig) GetWorm() bool {
 	return bool(s.Worm)
 }
 
-// SetCacheConfig sets the current cache config
-func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxuse int) {
-	s.Cache.Drives = drives
-	s.Cache.Exclude = exclude
-	s.Cache.Expiry = expiry
-	s.Cache.MaxUse = maxuse
-}
-
 // GetCacheConfig gets the current cache config
-func (s *serverConfig) GetCacheConfig() CacheConfig {
+func (s *serverConfig) GetCacheConfig() cache.Config {
 	if globalIsDiskCacheEnabled {
-		return CacheConfig{
+		return cache.Config{
 			Drives:  globalCacheDrives,
 			Exclude: globalCacheExcludes,
 			Expiry:  globalCacheExpiry,
@@ -153,7 +147,7 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
 		}
 	}
 	if s == nil {
-		return CacheConfig{}
+		return cache.Config{}
 	}
 	return s.Cache
 }
@@ -238,71 +232,126 @@ func (s *serverConfig) Validate() error {
 	return nil
 }
 
-// SetCompressionConfig sets the current compression config
-func (s *serverConfig) SetCompressionConfig(extensions []string, mimeTypes []string) {
-	s.Compression.Extensions = extensions
-	s.Compression.MimeTypes = mimeTypes
-	s.Compression.Enabled = globalIsCompressionEnabled
-}
-
-// GetCompressionConfig gets the current compression config
-func (s *serverConfig) GetCompressionConfig() compressionConfig {
-	return s.Compression
-}
-
-func (s *serverConfig) loadFromEnvs() {
+func (s *serverConfig) lookupConfigs() {
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
 		s.SetCredential(globalActiveCred)
+	} else {
+		globalActiveCred = s.GetCredential()
 	}
 
 	if globalIsEnvWORM {
 		s.SetWorm(globalWORMEnabled)
+	} else {
+		globalWORMEnabled = s.GetWorm()
 	}
 
 	if globalIsEnvRegion {
 		s.SetRegion(globalServerRegion)
+	} else {
+		globalServerRegion = s.GetRegion()
 	}
 
-	if globalIsStorageClass {
-		s.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
-	}
-
-	if globalIsDiskCacheEnabled {
-		s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
-	}
-
-	if err := Environment.LookupKMSConfig(s.KMS); err != nil {
-		logger.FatalIf(err, "Unable to setup the KMS")
-	}
-
-	if globalIsEnvCompression {
-		s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
-	}
-
-	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
-		u, err := xnet.ParseURL(jwksURL)
+	var err error
+	if globalIsXL {
+		s.StorageClass, err = storageclass.LookupConfig(s.StorageClass, globalXLSetDriveCount)
 		if err != nil {
-			logger.FatalIf(err, "Unable to parse MINIO_IAM_JWKS_URL %s", jwksURL)
+			logger.FatalIf(err, "Unable to initialize storage class config")
 		}
-		s.OpenID.JWKS.URL = u
 	}
 
-	if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
-		u, err := xnet.ParseURL(opaURL)
-		if err != nil {
-			logger.FatalIf(err, "Unable to parse MINIO_IAM_OPA_URL %s", opaURL)
-		}
-		opaArgs := iampolicy.OpaArgs{
-			URL:         u,
-			AuthToken:   os.Getenv("MINIO_IAM_OPA_AUTHTOKEN"),
-			Transport:   NewCustomHTTPTransport(),
-			CloseRespFn: xhttp.DrainBody,
-		}
-		logger.FatalIf(opaArgs.Validate(), "Unable to reach MINIO_IAM_OPA_URL %s", opaURL)
-		s.Policy.OPA.URL = opaArgs.URL
-		s.Policy.OPA.AuthToken = opaArgs.AuthToken
+	s.Cache, err = cache.LookupConfig(s.Cache)
+	if err != nil {
+		logger.FatalIf(err, "Unable to setup cache")
 	}
+
+	if len(s.Cache.Drives) > 0 {
+		globalIsDiskCacheEnabled = true
+		globalCacheDrives = s.Cache.Drives
+		globalCacheExcludes = s.Cache.Exclude
+		globalCacheExpiry = s.Cache.Expiry
+		globalCacheMaxUse = s.Cache.MaxUse
+
+		if cacheEncKey := env.Get(cache.EnvCacheEncryptionMasterKey, ""); cacheEncKey != "" {
+			globalCacheKMS, err = crypto.ParseMasterKey(cacheEncKey)
+			if err != nil {
+				logger.FatalIf(config.ErrInvalidCacheEncryptionKey(err),
+					"Unable to setup encryption cache")
+			}
+		}
+	}
+
+	s.KMS, err = crypto.LookupConfig(s.KMS)
+	if err != nil {
+		logger.FatalIf(err, "Unable to setup KMS config")
+	}
+
+	GlobalKMS, err = crypto.NewKMS(s.KMS)
+	if err != nil {
+		logger.FatalIf(err, "Unable to setup KMS with current KMS config")
+	}
+
+	globalAutoEncryption = strings.EqualFold(env.Get(crypto.EnvAutoEncryption, "off"), "on")
+	if globalAutoEncryption && GlobalKMS == nil {
+		logger.FatalIf(errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present"), "")
+	}
+
+	s.Compression, err = compress.LookupConfig(s.Compression)
+	if err != nil {
+		logger.FatalIf(err, "Unable to setup Compression")
+	}
+
+	if s.Compression.Enabled {
+		globalIsCompressionEnabled = s.Compression.Enabled
+		globalCompressExtensions = s.Compression.Extensions
+		globalCompressMimeTypes = s.Compression.MimeTypes
+	}
+
+	s.OpenID.JWKS, err = openid.LookupConfig(s.OpenID.JWKS, NewCustomHTTPTransport(), xhttp.DrainBody)
+	if err != nil {
+		logger.FatalIf(err, "Unable to initialize OpenID")
+	}
+
+	s.Policy.OPA, err = iampolicy.LookupConfig(s.Policy.OPA, NewCustomHTTPTransport(), xhttp.DrainBody)
+	if err != nil {
+		logger.FatalIf(err, "Unable to initialize OPA")
+	}
+
+	globalOpenIDValidators = getOpenIDValidators(s)
+	globalPolicyOPA = iampolicy.NewOpa(s.Policy.OPA)
+
+	s.LDAPServerConfig, err = xldap.Lookup(s.LDAPServerConfig, globalRootCAs)
+	if err != nil {
+		logger.FatalIf(err, "Unable to parse LDAP configuration from env")
+	}
+
+	// Load logger targets based on user's configuration
+	loggerUserAgent := getUserAgent(getMinioMode())
+
+	s.Logger, err = logger.LookupConfig(s.Logger)
+	if err != nil {
+		logger.FatalIf(err, "Unable to initialize logger")
+	}
+
+	for _, l := range s.Logger.HTTP {
+		if l.Enabled {
+			// Enable http logging
+			logger.AddTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
+		}
+	}
+
+	for _, l := range s.Logger.Audit {
+		if l.Enabled {
+			// Enable http audit logging
+			logger.AddAuditTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
+		}
+	}
+
+	if s.Logger.Console.Enabled {
+		// Enable console logging
+		logger.AddTarget(globalConsoleSys.Console())
+	}
+
 }
 
 // TestNotificationTargets tries to establish connections to all notification
@@ -313,7 +362,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewAMQPTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewAMQPTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("amqp(%s): %s", k, err.Error())
 		}
@@ -324,7 +373,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewElasticsearchTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewElasticsearchTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("elasticsearch(%s): %s", k, err.Error())
 		}
@@ -335,7 +384,10 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewKafkaTarget(k, v, GlobalServiceDoneCh)
+		if v.TLS.Enable {
+			v.TLS.RootCAs = globalRootCAs
+		}
+		t, err := target.NewKafkaTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("kafka(%s): %s", k, err.Error())
 		}
@@ -346,7 +398,8 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewMQTTTarget(k, v, GlobalServiceDoneCh)
+		v.RootCAs = globalRootCAs
+		t, err := target.NewMQTTTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("mqtt(%s): %s", k, err.Error())
 		}
@@ -357,7 +410,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewMySQLTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewMySQLTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("mysql(%s): %s", k, err.Error())
 		}
@@ -368,7 +421,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewNATSTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewNATSTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("nats(%s): %s", k, err.Error())
 		}
@@ -379,7 +432,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewNSQTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewNSQTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("nsq(%s): %s", k, err.Error())
 		}
@@ -390,7 +443,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewPostgreSQLTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewPostgreSQLTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("postgreSQL(%s): %s", k, err.Error())
 		}
@@ -401,7 +454,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewRedisTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewRedisTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("redis(%s): %s", k, err.Error())
 		}
@@ -412,56 +465,6 @@ func (s *serverConfig) TestNotificationTargets() error {
 	return nil
 }
 
-// Returns the string describing a difference with the given
-// configuration object. If the given configuration object is
-// identical, an empty string is returned.
-func (s *serverConfig) ConfigDiff(t *serverConfig) string {
-	switch {
-	case t == nil:
-		return "Given configuration is empty"
-	case s.Credential != t.Credential:
-		return "Credential configuration differs"
-	case s.Region != t.Region:
-		return "Region configuration differs"
-	case s.StorageClass != t.StorageClass:
-		return "StorageClass configuration differs"
-	case !reflect.DeepEqual(s.Cache, t.Cache):
-		return "Cache configuration differs"
-	case !reflect.DeepEqual(s.Compression, t.Compression):
-		return "Compression configuration differs"
-	case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
-		return "AMQP Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
-		return "NATS Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.NSQ, t.Notify.NSQ):
-		return "NSQ Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.Elasticsearch, t.Notify.Elasticsearch):
-		return "ElasticSearch Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.Redis, t.Notify.Redis):
-		return "Redis Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.PostgreSQL, t.Notify.PostgreSQL):
-		return "PostgreSQL Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.Kafka, t.Notify.Kafka):
-		return "Kafka Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.Webhook, t.Notify.Webhook):
-		return "Webhook Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.MySQL, t.Notify.MySQL):
-		return "MySQL Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.MQTT, t.Notify.MQTT):
-		return "MQTT Notification configuration differs"
-	case !reflect.DeepEqual(s.Logger, t.Logger):
-		return "Logger configuration differs"
-	case !reflect.DeepEqual(s.KMS, t.KMS):
-		return "KMS configuration differs"
-	case reflect.DeepEqual(s, t):
-		return ""
-	default:
-		// This case will not happen unless this comparison
-		// function has become stale.
-		return "Configuration differs"
-	}
-}
-
 func newServerConfig() *serverConfig {
 	cred, err := auth.GetNewCredentials()
 	logger.FatalIf(err, "")
@@ -470,111 +473,29 @@ func newServerConfig() *serverConfig {
 		Version:    serverConfigVersion,
 		Credential: cred,
 		Region:     globalMinioDefaultRegion,
-		StorageClass: storageClassConfig{
-			Standard: storageClass{},
-			RRS:      storageClass{},
+		StorageClass: storageclass.Config{
+			Standard: storageclass.StorageClass{},
+			RRS:      storageclass.StorageClass{},
 		},
-		Cache: CacheConfig{
+		Cache: cache.Config{
 			Drives:  []string{},
 			Exclude: []string{},
 			Expiry:  globalCacheExpiry,
 			MaxUse:  globalCacheMaxUse,
 		},
 		KMS:    crypto.KMSConfig{},
-		Notify: notifier{},
-		Compression: compressionConfig{
+		Notify: notify.NewConfig(),
+		Compression: compress.Config{
 			Enabled:    false,
 			Extensions: globalCompressExtensions,
 			MimeTypes:  globalCompressMimeTypes,
 		},
+		Logger: logger.NewConfig(),
 	}
 
-	// Make sure to initialize notification configs.
-	srvCfg.Notify.AMQP = make(map[string]target.AMQPArgs)
-	srvCfg.Notify.AMQP["1"] = target.AMQPArgs{}
-	srvCfg.Notify.MQTT = make(map[string]target.MQTTArgs)
-	srvCfg.Notify.MQTT["1"] = target.MQTTArgs{}
-	srvCfg.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
-	srvCfg.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{}
-	srvCfg.Notify.Redis = make(map[string]target.RedisArgs)
-	srvCfg.Notify.Redis["1"] = target.RedisArgs{}
-	srvCfg.Notify.NATS = make(map[string]target.NATSArgs)
-	srvCfg.Notify.NATS["1"] = target.NATSArgs{}
-	srvCfg.Notify.NSQ = make(map[string]target.NSQArgs)
-	srvCfg.Notify.NSQ["1"] = target.NSQArgs{}
-	srvCfg.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
-	srvCfg.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{}
-	srvCfg.Notify.MySQL = make(map[string]target.MySQLArgs)
-	srvCfg.Notify.MySQL["1"] = target.MySQLArgs{}
-	srvCfg.Notify.Kafka = make(map[string]target.KafkaArgs)
-	srvCfg.Notify.Kafka["1"] = target.KafkaArgs{}
-	srvCfg.Notify.Webhook = make(map[string]target.WebhookArgs)
-	srvCfg.Notify.Webhook["1"] = target.WebhookArgs{}
-
-	srvCfg.Cache.Drives = make([]string, 0)
-	srvCfg.Cache.Exclude = make([]string, 0)
-	srvCfg.Cache.Expiry = globalCacheExpiry
-	srvCfg.Cache.MaxUse = globalCacheMaxUse
-
-	// Console logging is on by default
-	srvCfg.Logger.Console.Enabled = true
-	// Create an example of HTTP logger
-	srvCfg.Logger.HTTP = make(map[string]loggerHTTP)
-	srvCfg.Logger.HTTP["target1"] = loggerHTTP{Endpoint: "https://username:password@example.com/api"}
-
 	return srvCfg
 }
 
-func (s *serverConfig) loadToCachedConfigs() {
-	if !globalIsEnvCreds {
-		globalActiveCred = s.GetCredential()
-	}
-	if !globalIsEnvWORM {
-		globalWORMEnabled = s.GetWorm()
-	}
-	if !globalIsEnvRegion {
-		globalServerRegion = s.GetRegion()
-	}
-	if !globalIsStorageClass {
-		globalStandardStorageClass, globalRRStorageClass = s.GetStorageClass()
-	}
-	if !globalIsDiskCacheEnabled {
-		cacheConf := s.GetCacheConfig()
-		globalCacheDrives = cacheConf.Drives
-		globalCacheExcludes = cacheConf.Exclude
-		globalCacheExpiry = cacheConf.Expiry
-		globalCacheMaxUse = cacheConf.MaxUse
-	}
-	if err := Environment.LookupKMSConfig(s.KMS); err != nil {
-		logger.FatalIf(err, "Unable to setup the KMS %s", s.KMS.Vault.Endpoint)
-	}
-
-	if !globalIsCompressionEnabled {
-		compressionConf := s.GetCompressionConfig()
-		globalCompressExtensions = compressionConf.Extensions
-		globalCompressMimeTypes = compressionConf.MimeTypes
-		globalIsCompressionEnabled = compressionConf.Enabled
-	}
-
-	if s.OpenID.JWKS.URL != nil && s.OpenID.JWKS.URL.String() != "" {
-		logger.FatalIf(s.OpenID.JWKS.PopulatePublicKey(),
-			"Unable to populate public key from JWKS URL %s", s.OpenID.JWKS.URL)
-	}
-
-	globalIAMValidators = getAuthValidators(s)
-
-	if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
-		opaArgs := iampolicy.OpaArgs{
-			URL:         s.Policy.OPA.URL,
-			AuthToken:   s.Policy.OPA.AuthToken,
-			Transport:   NewCustomHTTPTransport(),
-			CloseRespFn: xhttp.DrainBody,
-		}
-		logger.FatalIf(opaArgs.Validate(), "Unable to reach OPA URL %s", s.Policy.OPA.URL)
-		globalPolicyOPA = iampolicy.NewOpa(opaArgs)
-	}
-}
-
 // newSrvConfig - initialize a new server config, saves env parameters if
 // found, otherwise use default parameters
 func newSrvConfig(objAPI ObjectLayer) error {
@@ -582,10 +503,7 @@ func newSrvConfig(objAPI ObjectLayer) error {
 	srvCfg := newServerConfig()
 
 	// Override any values from ENVs.
-	srvCfg.loadFromEnvs()
-
-	// Load values to cached global values.
-	srvCfg.loadToCachedConfigs()
+	srvCfg.lookupConfigs()
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()
@@ -611,14 +529,11 @@ func getValidConfig(objAPI ObjectLayer) (*serverConfig, error) {
 func loadConfig(objAPI ObjectLayer) error {
 	srvCfg, err := getValidConfig(objAPI)
 	if err != nil {
-		return uiErrInvalidConfig(nil).Msg(err.Error())
+		return config.ErrInvalidConfig(nil).Msg(err.Error())
 	}
 
 	// Override any values from ENVs.
-	srvCfg.loadFromEnvs()
-
-	// Load values to cached global values.
-	srvCfg.loadToCachedConfigs()
+	srvCfg.lookupConfigs()
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()
@@ -628,15 +543,15 @@ func loadConfig(objAPI ObjectLayer) error {
 	return nil
 }
 
-// getAuthValidators - returns ValidatorList which contains
+// getOpenIDValidators - returns ValidatorList which contains
 // enabled providers in server config.
 // A new authentication provider is added like below
-// * Add a new provider in pkg/iam/validator package.
-func getAuthValidators(config *serverConfig) *validator.Validators {
-	validators := validator.NewValidators()
+// * Add a new provider in pkg/iam/openid package.
+func getOpenIDValidators(config *serverConfig) *openid.Validators {
+	validators := openid.NewValidators()
 
 	if config.OpenID.JWKS.URL != nil {
-		validators.Add(validator.NewJWT(config.OpenID.JWKS))
+		validators.Add(openid.NewJWT(config.OpenID.JWKS))
 	}
 
 	return validators
@@ -654,7 +569,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 	}
 	for id, args := range config.Notify.AMQP {
 		if args.Enable {
-			newTarget, err := target.NewAMQPTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewAMQPTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -668,7 +583,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Elasticsearch {
 		if args.Enable {
-			newTarget, err := target.NewElasticsearchTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewElasticsearchTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -684,7 +599,10 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Kafka {
 		if args.Enable {
-			newTarget, err := target.NewKafkaTarget(id, args, GlobalServiceDoneCh)
+			if args.TLS.Enable {
+				args.TLS.RootCAs = globalRootCAs
+			}
+			newTarget, err := target.NewKafkaTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -699,7 +617,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 	for id, args := range config.Notify.MQTT {
 		if args.Enable {
 			args.RootCAs = globalRootCAs
-			newTarget, err := target.NewMQTTTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewMQTTTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -713,7 +631,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.MySQL {
 		if args.Enable {
-			newTarget, err := target.NewMySQLTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewMySQLTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -727,7 +645,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.NATS {
 		if args.Enable {
-			newTarget, err := target.NewNATSTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewNATSTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -741,7 +659,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.NSQ {
 		if args.Enable {
-			newTarget, err := target.NewNSQTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewNSQTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -755,7 +673,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.PostgreSQL {
 		if args.Enable {
-			newTarget, err := target.NewPostgreSQLTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewPostgreSQLTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -769,7 +687,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Redis {
 		if args.Enable {
-			newTarget, err := target.NewRedisTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewRedisTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -784,7 +702,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 	for id, args := range config.Notify.Webhook {
 		if args.Enable {
 			args.RootCAs = globalRootCAs
-			newTarget := target.NewWebhookTarget(id, args, GlobalServiceDoneCh)
+			newTarget := target.NewWebhookTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err := targetList.Add(newTarget); err != nil {
 				logger.LogIf(context.Background(), err)
 				continue

cmd/config-current_test.go

@@ -21,9 +21,6 @@ import (
 	"os"
 	"path"
 	"testing"
-
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/event/target"
 )
 
 func TestServerConfig(t *testing.T) {
@@ -62,79 +59,6 @@ func TestServerConfig(t *testing.T) {
 	}
 }
 
-func TestServerConfigWithEnvs(t *testing.T) {
-
-	os.Setenv("MINIO_BROWSER", "off")
-	defer os.Unsetenv("MINIO_BROWSER")
-
-	os.Setenv("MINIO_WORM", "on")
-	defer os.Unsetenv("MINIO_WORM")
-
-	os.Setenv("MINIO_ACCESS_KEY", "minio")
-	defer os.Unsetenv("MINIO_ACCESS_KEY")
-
-	os.Setenv("MINIO_SECRET_KEY", "minio123")
-	defer os.Unsetenv("MINIO_SECRET_KEY")
-
-	os.Setenv("MINIO_REGION", "us-west-1")
-	defer os.Unsetenv("MINIO_REGION")
-
-	os.Setenv("MINIO_DOMAIN", "domain.com")
-	defer os.Unsetenv("MINIO_DOMAIN")
-
-	defer resetGlobalIsEnvs()
-
-	objLayer, fsDir, err := prepareFS()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(fsDir)
-
-	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
-		t.Fatalf("Init Test config failed")
-	}
-
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = objLayer
-	globalObjLayerMutex.Unlock()
-
-	serverHandleEnvVars()
-
-	// Init config
-	initConfig(objLayer)
-
-	// Check if serverConfig has browser disabled
-	if globalIsBrowserEnabled {
-		t.Error("Expected browser to be disabled but it is not")
-	}
-
-	// Check if serverConfig returns WORM config from the env
-	if !globalServerConfig.GetWorm() {
-		t.Error("Expected WORM to be enabled but it is not")
-	}
-
-	// Check if serverConfig has region from the environment
-	if globalServerConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expected region to be \"us-west-1\", found %v", globalServerConfig.GetRegion())
-	}
-
-	// Check if serverConfig has credentials from the environment
-	cred := globalServerConfig.GetCredential()
-
-	if cred.AccessKey != "minio" {
-		t.Errorf("Expected access key to be `minio`, found %s", cred.AccessKey)
-	}
-
-	if cred.SecretKey != "minio123" {
-		t.Errorf("Expected access key to be `minio123`, found %s", cred.SecretKey)
-	}
-
-	// Check if serverConfig has the correct domain
-	if globalDomainNames[0] != "domain.com" {
-		t.Errorf("Expected Domain to be `domain.com`, found " + globalDomainNames[0])
-	}
-}
-
 // Tests config validator..
 func TestValidateConfig(t *testing.T) {
 	objLayer, fsDir, err := prepareFS()
@@ -253,114 +177,3 @@ func TestValidateConfig(t *testing.T) {
 	}
 
 }
-
-func TestConfigDiff(t *testing.T) {
-	testCases := []struct {
-		s, t *serverConfig
-		diff string
-	}{
-		// 1
-		{&serverConfig{}, nil, "Given configuration is empty"},
-		// 2
-		{
-			&serverConfig{Credential: auth.Credentials{
-				AccessKey:  "u1",
-				SecretKey:  "p1",
-				Expiration: timeSentinel,
-			}},
-			&serverConfig{Credential: auth.Credentials{
-				AccessKey:  "u1",
-				SecretKey:  "p2",
-				Expiration: timeSentinel,
-			}},
-			"Credential configuration differs",
-		},
-		// 3
-		{&serverConfig{Region: "us-east-1"}, &serverConfig{Region: "us-west-1"}, "Region configuration differs"},
-		// 4
-		{
-			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 6}}},
-			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 4}}},
-			"StorageClass configuration differs",
-		},
-		// 5
-		{
-			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: false}}}},
-			"AMQP Notification configuration differs",
-		},
-		// 6
-		{
-			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: false}}}},
-			"NATS Notification configuration differs",
-		},
-		// 7
-		{
-			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: false}}}},
-			"NSQ Notification configuration differs",
-		},
-		// 8
-		{
-			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: false}}}},
-			"ElasticSearch Notification configuration differs",
-		},
-		// 9
-		{
-			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: false}}}},
-			"Redis Notification configuration differs",
-		},
-		// 10
-		{
-			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: false}}}},
-			"PostgreSQL Notification configuration differs",
-		},
-		// 11
-		{
-			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: false}}}},
-			"Kafka Notification configuration differs",
-		},
-		// 12
-		{
-			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: false}}}},
-			"Webhook Notification configuration differs",
-		},
-		// 13
-		{
-			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: false}}}},
-			"MySQL Notification configuration differs",
-		},
-		// 14
-		{
-			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
-			"MQTT Notification configuration differs",
-		},
-		// 15
-		{
-			&serverConfig{Logger: loggerConfig{
-				Console: loggerConsole{Enabled: true},
-				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address1"}},
-			}},
-			&serverConfig{Logger: loggerConfig{
-				Console: loggerConsole{Enabled: true},
-				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address2"}},
-			}},
-			"Logger configuration differs",
-		},
-	}
-
-	for i, testCase := range testCases {
-		got := testCase.s.ConfigDiff(testCase.t)
-		if got != testCase.diff {
-			t.Errorf("Test %d: got %s expected %s", i+1, got, testCase.diff)
-		}
-	}
-}

cmd/config-migrate.go

@@ -29,8 +29,8 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/openid"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/iam/validator"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
 )
@@ -65,7 +65,7 @@ func migrateConfig() error {
 	// Load only config version information.
 	version, err := GetVersion(getConfigFile())
 	if err != nil {
-		if os.IsNotExist(err) {
+		if os.IsNotExist(err) || os.IsPermission(err) {
 			return nil
 		}
 		return err
@@ -234,7 +234,7 @@ func purgeV1() error {
 
 	cv1 := &configV1{}
 	_, err := Load(configFile, cv1)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
@@ -255,7 +255,7 @@ func migrateV2ToV3() error {
 
 	cv2 := &configV2{}
 	_, err := Load(configFile, cv2)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘2’. %v", err)
@@ -314,7 +314,7 @@ func migrateV3ToV4() error {
 
 	cv3 := &configV3{}
 	_, err := Load(configFile, cv3)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘3’. %v", err)
@@ -352,7 +352,7 @@ func migrateV4ToV5() error {
 
 	cv4 := &configV4{}
 	_, err := Load(configFile, cv4)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘4’. %v", err)
@@ -393,7 +393,7 @@ func migrateV5ToV6() error {
 
 	cv5 := &configV5{}
 	_, err := Load(configFile, cv5)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘5’. %v", err)
@@ -482,7 +482,7 @@ func migrateV6ToV7() error {
 
 	cv6 := &configV6{}
 	_, err := Load(configFile, cv6)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘6’. %v", err)
@@ -538,7 +538,7 @@ func migrateV7ToV8() error {
 
 	cv7 := &serverConfigV7{}
 	_, err := Load(configFile, cv7)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘7’. %v", err)
@@ -600,7 +600,7 @@ func migrateV8ToV9() error {
 
 	cv8 := &serverConfigV8{}
 	_, err := Load(configFile, cv8)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘8’. %v", err)
@@ -670,7 +670,7 @@ func migrateV9ToV10() error {
 
 	cv9 := &serverConfigV9{}
 	_, err := Load(configFile, cv9)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘9’. %v", err)
@@ -738,7 +738,7 @@ func migrateV10ToV11() error {
 
 	cv10 := &serverConfigV10{}
 	_, err := Load(configFile, cv10)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘10’. %v", err)
@@ -809,7 +809,7 @@ func migrateV11ToV12() error {
 
 	cv11 := &serverConfigV11{}
 	_, err := Load(configFile, cv11)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘11’. %v", err)
@@ -906,7 +906,7 @@ func migrateV12ToV13() error {
 
 	cv12 := &serverConfigV12{}
 	_, err := Load(configFile, cv12)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘12’. %v", err)
@@ -986,7 +986,7 @@ func migrateV13ToV14() error {
 
 	cv13 := &serverConfigV13{}
 	_, err := Load(configFile, cv13)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘13’. %v", err)
@@ -1071,7 +1071,7 @@ func migrateV14ToV15() error {
 
 	cv14 := &serverConfigV14{}
 	_, err := Load(configFile, cv14)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘14’. %v", err)
@@ -1161,7 +1161,7 @@ func migrateV15ToV16() error {
 
 	cv15 := &serverConfigV15{}
 	_, err := Load(configFile, cv15)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘15’. %v", err)
@@ -1251,7 +1251,7 @@ func migrateV16ToV17() error {
 
 	cv16 := &serverConfigV16{}
 	_, err := Load(configFile, cv16)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘16’. %v", err)
@@ -1372,7 +1372,7 @@ func migrateV17ToV18() error {
 
 	cv17 := &serverConfigV17{}
 	_, err := Load(configFile, cv17)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘17’. %v", err)
@@ -1474,7 +1474,7 @@ func migrateV18ToV19() error {
 
 	cv18 := &serverConfigV18{}
 	_, err := Load(configFile, cv18)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
@@ -1580,7 +1580,7 @@ func migrateV19ToV20() error {
 
 	cv19 := &serverConfigV19{}
 	_, err := Load(configFile, cv19)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
@@ -1685,7 +1685,7 @@ func migrateV20ToV21() error {
 
 	cv20 := &serverConfigV20{}
 	_, err := Load(configFile, cv20)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘20’. %v", err)
@@ -1789,7 +1789,7 @@ func migrateV21ToV22() error {
 
 	cv21 := &serverConfigV21{}
 	_, err := Load(configFile, cv21)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘21’. %v", err)
@@ -1893,7 +1893,7 @@ func migrateV22ToV23() error {
 
 	cv22 := &serverConfigV22{}
 	_, err := Load(configFile, cv22)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘22’. %v", err)
@@ -2006,7 +2006,7 @@ func migrateV23ToV24() error {
 
 	cv23 := &serverConfigV23{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv23)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘23’. %v", err)
@@ -2119,7 +2119,7 @@ func migrateV24ToV25() error {
 
 	cv24 := &serverConfigV24{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv24)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘24’. %v", err)
@@ -2237,7 +2237,7 @@ func migrateV25ToV26() error {
 
 	cv25 := &serverConfigV25{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘25’. %v", err)
@@ -2359,7 +2359,7 @@ func migrateV26ToV27() error {
 	// in the new `logger` field
 	srvConfig := &serverConfigV27{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
@@ -2373,8 +2373,8 @@ func migrateV26ToV27() error {
 	// Enable console logging by default to avoid breaking users
 	// current deployments
 	srvConfig.Logger.Console.Enabled = true
-	srvConfig.Logger.HTTP = make(map[string]loggerHTTP)
-	srvConfig.Logger.HTTP["1"] = loggerHTTP{}
+	srvConfig.Logger.HTTP = make(map[string]logger.HTTP)
+	srvConfig.Logger.HTTP["1"] = logger.HTTP{}
 
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %v", err)
@@ -2392,7 +2392,7 @@ func migrateV27ToV28() error {
 
 	srvConfig := &serverConfigV28{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
@@ -2459,14 +2459,17 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	var config = &serverConfig{}
 	for _, cfgFile := range configFiles {
 		if _, err = Load(cfgFile, config); err != nil {
-			if !os.IsNotExist(err) {
+			if !os.IsNotExist(err) && !os.IsPermission(err) {
 				return err
 			}
 			continue
 		}
 		break
 	}
-	if os.IsNotExist(err) {
+	if os.IsPermission(err) {
+		logger.Info("Older config found but not readable %s, proceeding to initialize new config anyways", err)
+	}
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		// Initialize the server config, if no config exists.
 		return newSrvConfig(objAPI)
 	}
@@ -2654,7 +2657,7 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
 	}
 
 	cfg.Version = "31"
-	cfg.OpenID.JWKS = validator.JWKSArgs{
+	cfg.OpenID.JWKS = openid.JWKSArgs{
 		URL: &xnet.URL{},
 	}
 	cfg.Policy.OPA = iampolicy.OpaArgs{

cmd/config-versions.go

@@ -19,11 +19,18 @@ package cmd
 import (
 	"sync"
 
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/cache"
+	"github.com/minio/minio/cmd/config/compress"
+	xldap "github.com/minio/minio/cmd/config/ldap"
+	"github.com/minio/minio/cmd/config/notify"
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/openid"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -265,7 +272,7 @@ type serverConfigV7 struct {
 	Notify notifierV1 `json:"notify"`
 }
 
-// serverConfigV8 server configuration version '8'. Adds NATS notifier
+// serverConfigV8 server configuration version '8'. Adds NATS notify.Config
 // configuration.
 type serverConfigV8 struct {
 	Version string `json:"version"`
@@ -282,7 +289,7 @@ type serverConfigV8 struct {
 }
 
 // serverConfigV9 server configuration version '9'. Adds PostgreSQL
-// notifier configuration.
+// notify.Config configuration.
 type serverConfigV9 struct {
 	Version string `json:"version"`
 
@@ -400,7 +407,7 @@ type serverConfigV14 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -417,7 +424,7 @@ type serverConfigV15 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -455,7 +462,7 @@ type serverConfigV16 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -474,7 +481,7 @@ type serverConfigV17 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -493,7 +500,7 @@ type serverConfigV18 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -511,7 +518,7 @@ type serverConfigV19 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -529,7 +536,7 @@ type serverConfigV20 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Additional error logging configuration.
@@ -547,7 +554,7 @@ type serverConfigV21 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Notification queue configuration.
@@ -556,43 +563,37 @@ type serverConfigV21 struct {
 
 // serverConfigV22 is just like version '21' with added support
 // for StorageClass.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV22 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV23 is just like version '22' with addition of cache field.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV23 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
@@ -600,23 +601,20 @@ type serverConfigV23 struct {
 
 // serverConfigV24 is just like version '23', we had to revert
 // the changes which were made in 6fb06045028b7a57c37c60a612c8e50735279ab4
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV24 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
@@ -624,9 +622,6 @@ type serverConfigV24 struct {
 
 // serverConfigV25 is just like version '24', stores additionally
 // worm variable.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV25 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -635,22 +630,22 @@ type serverConfigV25 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
-	Worm       BoolFlag         `json:"worm"`
+	Browser    config.BoolFlag  `json:"browser"`
+	Worm       config.BoolFlag  `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV26 is just like version '25', stores additionally
-// cache max use value in 'CacheConfig'.
+// cache max use value in 'cache.Config'.
 type serverConfigV26 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -659,39 +654,22 @@ type serverConfigV26 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
-	Worm       BoolFlag         `json:"worm"`
+	Browser    config.BoolFlag  `json:"browser"`
+	Worm       config.BoolFlag  `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
 
-type loggerConsole struct {
-	Enabled bool `json:"enabled"`
-}
-
-type loggerHTTP struct {
-	Enabled  bool   `json:"enabled"`
-	Endpoint string `json:"endpoint"`
-}
-
-type loggerConfig struct {
-	Console loggerConsole         `json:"console"`
-	HTTP    map[string]loggerHTTP `json:"http"`
-}
-
 // serverConfigV27 is just like version '26', stores additionally
 // the logger field
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV27 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -700,28 +678,25 @@ type serverConfigV27 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
-	Worm       BoolFlag         `json:"worm"`
+	Browser    config.BoolFlag  `json:"browser"`
+	Worm       config.BoolFlag  `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 }
 
 // serverConfigV28 is just like version '27', additionally
 // storing KMS config
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV28 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -730,13 +705,13 @@ type serverConfigV28 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
@@ -745,19 +720,12 @@ type serverConfigV28 struct {
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 }
 
 // serverConfigV29 is just like version '28'.
 type serverConfigV29 serverConfigV28
 
-// compressionConfig represents the compression settings.
-type compressionConfig struct {
-	Enabled    bool     `json:"enabled"`
-	Extensions []string `json:"extensions"`
-	MimeTypes  []string `json:"mime-types"`
-}
-
 // serverConfigV30 is just like version '29', stores additionally
 // extensions and mimetypes fields for compression.
 type serverConfigV30 struct {
@@ -766,13 +734,13 @@ type serverConfigV30 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
@@ -781,10 +749,10 @@ type serverConfigV30 struct {
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 }
 
 // serverConfigV31 is just like version '30', with OPA and OpenID configuration.
@@ -794,13 +762,13 @@ type serverConfigV31 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
@@ -809,15 +777,15 @@ type serverConfigV31 struct {
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 
 	// OpenID configuration
 	OpenID struct {
 		// JWKS validator config.
-		JWKS validator.JWKSArgs `json:"jwks"`
+		JWKS openid.JWKSArgs `json:"jwks"`
 	} `json:"openid"`
 
 	// External policy enforcements.
@@ -829,19 +797,6 @@ type serverConfigV31 struct {
 	} `json:"policy"`
 }
 
-type notifier struct {
-	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
-	Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
-	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
-	MQTT          map[string]target.MQTTArgs          `json:"mqtt"`
-	MySQL         map[string]target.MySQLArgs         `json:"mysql"`
-	NATS          map[string]target.NATSArgs          `json:"nats"`
-	NSQ           map[string]target.NSQArgs           `json:"nsq"`
-	PostgreSQL    map[string]target.PostgreSQLArgs    `json:"postgresql"`
-	Redis         map[string]target.RedisArgs         `json:"redis"`
-	Webhook       map[string]target.WebhookArgs       `json:"webhook"`
-}
-
 // serverConfigV32 is just like version '31' with added nsq notifer.
 type serverConfigV32 struct {
 	Version string `json:"version"`
@@ -849,30 +804,30 @@ type serverConfigV32 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notify.Config `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 
 	// OpenID configuration
 	OpenID struct {
 		// JWKS validator config.
-		JWKS validator.JWKSArgs `json:"jwks"`
+		JWKS openid.JWKSArgs `json:"jwks"`
 	} `json:"openid"`
 
 	// External policy enforcements.
@@ -893,30 +848,30 @@ type serverConfigV33 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notify.Config `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 
 	// OpenID configuration
 	OpenID struct {
 		// JWKS validator config.
-		JWKS validator.JWKSArgs `json:"jwks"`
+		JWKS openid.JWKSArgs `json:"jwks"`
 	} `json:"openid"`
 
 	// External policy enforcements.
@@ -926,4 +881,6 @@ type serverConfigV33 struct {
 
 		// Add new external policy enforcements here.
 	} `json:"policy"`
+
+	LDAPServerConfig xldap.Config `json:"ldapserverconfig"`
 }

cmd/config.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"path"
 	"runtime"
 	"strings"
@@ -136,18 +137,23 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	//    of the object layer.
 	//  - Write quorum not met when upgrading configuration
 	//    version is needed.
-	for range newRetryTimerSimple(doneCh) {
-		if err := initConfig(objAPI); err != nil {
-			if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-				logger.Info("Waiting for configuration to be initialized..")
-				continue
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case <-retryTimerCh:
+			if err := initConfig(objAPI); err != nil {
+				if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for configuration to be initialized..")
+					continue
+				}
+				return err
 			}
-			return err
+			return nil
+		case <-globalOSSignalCh:
+			return fmt.Errorf("Initializing config sub-system gracefully stopped")
 		}
-		break
 	}
-	return nil
 }
 
 // NewConfigSys - creates new config system object.

cmd/config/bool-flag.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"encoding/json"

cmd/config/bool-flag_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"errors"

cmd/config/cache/config.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package cache
 
 import (
 	"encoding/json"
@@ -22,11 +22,12 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/ellipses"
 )
 
-// CacheConfig represents cache config settings
-type CacheConfig struct {
+// Config represents cache config settings
+type Config struct {
 	Drives  []string `json:"drives"`
 	Expiry  int      `json:"expiry"`
 	MaxUse  int      `json:"maxuse"`
@@ -35,8 +36,8 @@ type CacheConfig struct {
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
 // json entries for CacheConfig.
-func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
-	type Alias CacheConfig
+func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
+	type Alias Config
 	var _cfg = &struct {
 		*Alias
 	}{
@@ -83,7 +84,7 @@ func parseCacheDrives(drives []string) ([]string, error) {
 
 	for _, d := range endpoints {
 		if !filepath.IsAbs(d) {
-			return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
+			return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
 		}
 	}
 	return endpoints, nil
@@ -93,7 +94,7 @@ func parseCacheDrives(drives []string) ([]string, error) {
 func parseCacheDrivePaths(arg string) (ep []string, err error) {
 	patterns, perr := ellipses.FindEllipsesPatterns(arg)
 	if perr != nil {
-		return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
+		return []string{}, config.ErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
 	}
 
 	for _, lbls := range patterns.Expand() {
@@ -107,10 +108,10 @@ func parseCacheDrivePaths(arg string) (ep []string, err error) {
 func parseCacheExcludes(excludes []string) ([]string, error) {
 	for _, e := range excludes {
 		if len(e) == 0 {
-			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
+			return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
 		}
-		if hasPrefix(e, SlashSeparator) {
-			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
+		if strings.HasPrefix(e, "/") {
+			return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
 		}
 	}
 	return excludes, nil

cmd/config/cache/config_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package cache
 
 import (
 	"reflect"
@@ -35,7 +35,7 @@ func TestParseCacheDrives(t *testing.T) {
 		{"bucket1/*;*.png;images/trip/barcelona/*", []string{}, false},
 		{"bucket1", []string{}, false},
 	}
-	if runtime.GOOS == globalWindowsOSName {
+	if runtime.GOOS == "windows" {
 		testCases = append(testCases, struct {
 			driveStr         string
 			expectedPatterns []string

cmd/config/cache/lookup.go

@@ -0,0 +1,79 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cache
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Cache ENVs
+const (
+	EnvCacheDrives              = "MINIO_CACHE_DRIVES"
+	EnvCacheExclude             = "MINIO_CACHE_EXCLUDE"
+	EnvCacheExpiry              = "MINIO_CACHE_EXPIRY"
+	EnvCacheMaxUse              = "MINIO_CACHE_MAXUSE"
+	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
+)
+
+const (
+	cacheEnvDelimiter = ";"
+)
+
+// LookupConfig - extracts cache configuration provided by environment
+// variables and merge them with provided CacheConfiguration.
+func LookupConfig(cfg Config) (Config, error) {
+	if drives := env.Get(EnvCacheDrives, strings.Join(cfg.Drives, ",")); drives != "" {
+		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
+		if err != nil {
+			return cfg, err
+		}
+		cfg.Drives = driveList
+	}
+
+	if excludes := env.Get(EnvCacheExclude, strings.Join(cfg.Exclude, ",")); excludes != "" {
+		excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter))
+		if err != nil {
+			return cfg, err
+		}
+		cfg.Exclude = excludeList
+	}
+
+	if expiryStr := env.Get(EnvCacheExpiry, strconv.Itoa(cfg.Expiry)); expiryStr != "" {
+		expiry, err := strconv.Atoi(expiryStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheExpiryValue(err)
+		}
+		cfg.Expiry = expiry
+	}
+
+	if maxUseStr := env.Get(EnvCacheMaxUse, strconv.Itoa(cfg.MaxUse)); maxUseStr != "" {
+		maxUse, err := strconv.Atoi(maxUseStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheMaxUse(err)
+		}
+		// maxUse should be a valid percentage.
+		if maxUse > 0 && maxUse <= 100 {
+			cfg.MaxUse = maxUse
+		}
+	}
+
+	return cfg, nil
+}

cmd/config/certs.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"bytes"
@@ -25,16 +25,18 @@ import (
 	"encoding/pem"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
-	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/env"
 )
 
-// TLSPrivateKeyPassword is the environment variable which contains the password used
+// EnvCertPassword is the environment variable which contains the password used
 // to decrypt the TLS private key. It must be set if the TLS private key is
 // password protected.
-const TLSPrivateKeyPassword = "MINIO_CERT_PASSWD"
+const EnvCertPassword = "MINIO_CERT_PASSWD"
 
-func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
+// ParsePublicCertFile - parses public cert into its *x509.Certificate equivalent.
+func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
 	// Read certificate file.
 	var data []byte
 	if data, err = ioutil.ReadFile(certFile); err != nil {
@@ -49,25 +51,27 @@ func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 	for len(current) > 0 {
 		var pemBlock *pem.Block
 		if pemBlock, current = pem.Decode(current); pemBlock == nil {
-			return nil, uiErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
+			return nil, ErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
 		}
 
 		var x509Cert *x509.Certificate
 		if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
-			return nil, uiErrSSLUnexpectedData(err)
+			return nil, ErrSSLUnexpectedData(err)
 		}
 
 		x509Certs = append(x509Certs, x509Cert)
 	}
 
 	if len(x509Certs) == 0 {
-		return nil, uiErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
+		return nil, ErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
 	}
 
 	return x509Certs, nil
 }
 
-func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
+// GetRootCAs - returns all the root CAs into certPool
+// at the input certsCADir
+func GetRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	rootCAs, _ := x509.SystemCertPool()
 	if rootCAs == nil {
 		// In some systems (like Windows) system cert pool is
@@ -76,9 +80,9 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 		rootCAs = x509.NewCertPool()
 	}
 
-	fis, err := readDir(certsCAsDir)
+	fis, err := ioutil.ReadDir(certsCAsDir)
 	if err != nil {
-		if err == errFileNotFound {
+		if os.IsNotExist(err) {
 			err = nil // Return success if CA's directory is missing.
 		}
 		return rootCAs, err
@@ -86,76 +90,61 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 
 	// Load all custom CA files.
 	for _, fi := range fis {
-		// Skip all directories.
-		if hasSuffix(fi, SlashSeparator) {
-			continue
+		// Only load regular files as public cert.
+		if fi.Mode().IsRegular() {
+			caCert, err := ioutil.ReadFile(filepath.Join(certsCAsDir, fi.Name()))
+			if err != nil {
+				return rootCAs, err
+			}
+			rootCAs.AppendCertsFromPEM(caCert)
 		}
-		caCert, err := ioutil.ReadFile(pathJoin(certsCAsDir, fi))
-		if err != nil {
-			return rootCAs, err
-		}
-		rootCAs.AppendCertsFromPEM(caCert)
 	}
 	return rootCAs, nil
 }
 
-// load an X509 key pair (private key , certificate) from the provided
-// paths. The private key may be encrypted and is decrypted using the
-// ENV_VAR: MINIO_CERT_PASSWD.
-func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
+// LoadX509KeyPair - load an X509 key pair (private key , certificate)
+// from the provided paths. The private key may be encrypted and is
+// decrypted using the ENV_VAR: MINIO_CERT_PASSWD.
+func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	certPEMBlock, err := ioutil.ReadFile(certFile)
 	if err != nil {
-		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
+		return tls.Certificate{}, ErrSSLUnexpectedError(err)
 	}
 	keyPEMBlock, err := ioutil.ReadFile(keyFile)
 	if err != nil {
-		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
+		return tls.Certificate{}, ErrSSLUnexpectedError(err)
 	}
 	key, rest := pem.Decode(keyPEMBlock)
 	if len(rest) > 0 {
-		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
+		return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
 	}
 	if x509.IsEncryptedPEMBlock(key) {
-		password, ok := os.LookupEnv(TLSPrivateKeyPassword)
+		password, ok := env.Lookup(EnvCertPassword)
 		if !ok {
-			return tls.Certificate{}, uiErrSSLNoPassword(nil)
+			return tls.Certificate{}, ErrSSLNoPassword(nil)
 		}
 		decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
 		if decErr != nil {
-			return tls.Certificate{}, uiErrSSLWrongPassword(decErr)
+			return tls.Certificate{}, ErrSSLWrongPassword(decErr)
 		}
 		keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
 	}
 	cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
 	if err != nil {
-		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg(err.Error())
+		return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg(err.Error())
 	}
 	// Ensure that the private key is not a P-384 or P-521 EC key.
 	// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
 	if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
 		if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
-			if name := pub.Params().Name; name == "P-384" || name == "P-521" { // unfortunately there is no cleaner way to check
-				return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
+			switch pub.Params().Name {
+			case "P-384":
+				fallthrough
+			case "P-521":
+				// unfortunately there is no cleaner way to check
+				return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", pub.Params().Name)
 			}
 		}
 	}
 	return cert, nil
 }
-
-func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
-	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
-		return nil, nil, false, nil
-	}
-
-	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
-		return nil, nil, false, err
-	}
-
-	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
-	if err != nil {
-		return nil, nil, false, err
-	}
-
-	secureConn = true
-	return x509Certs, c, secureConn, nil
-}

cmd/config/certs_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"fmt"
@@ -176,7 +176,7 @@ M9ofSEt/bdRD
 	}
 
 	for _, testCase := range testCases {
-		certs, err := parsePublicCertFile(testCase.certFile)
+		certs, err := ParsePublicCertFile(testCase.certFile)
 
 		if testCase.expectedErr == nil {
 			if err != nil {
@@ -234,7 +234,7 @@ func TestGetRootCAs(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
-		_, err := getRootCAs(testCase.certCAsDir)
+		_, err := GetRootCAs(testCase.certCAsDir)
 
 		if testCase.expectedErr == nil {
 			if err != nil {
@@ -260,11 +260,11 @@ func TestLoadX509KeyPair(t *testing.T) {
 			t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err)
 		}
 
-		os.Unsetenv(TLSPrivateKeyPassword)
+		os.Unsetenv(EnvCertPassword)
 		if testCase.password != "" {
-			os.Setenv(TLSPrivateKeyPassword, testCase.password)
+			os.Setenv(EnvCertPassword, testCase.password)
 		}
-		_, err = loadX509KeyPair(certificate, privateKey)
+		_, err = LoadX509KeyPair(certificate, privateKey)
 		if err != nil && !testCase.shouldFail {
 			t.Errorf("Test %d: test should succeed but it failed: %v", i, err)
 		}

cmd/config/compress/compress.go

@@ -0,0 +1,78 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package compress
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Config represents the compression settings.
+type Config struct {
+	Enabled    bool     `json:"enabled"`
+	Extensions []string `json:"extensions"`
+	MimeTypes  []string `json:"mime-types"`
+}
+
+// Compression environment variables
+const (
+	EnvCompress           = "MINIO_COMPRESS"
+	EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressMimeTypes  = "MINIO_COMPRESS_MIMETYPES"
+)
+
+// Parses the given compression exclude list `extensions` or `content-types`.
+func parseCompressIncludes(includes []string) ([]string, error) {
+	for _, e := range includes {
+		if len(e) == 0 {
+			return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
+		}
+	}
+	return includes, nil
+}
+
+// LookupConfig - lookup compression config.
+func LookupConfig(cfg Config) (Config, error) {
+	if compress := env.Get(EnvCompress, strconv.FormatBool(cfg.Enabled)); compress != "" {
+		cfg.Enabled = strings.EqualFold(compress, "true")
+	}
+
+	compressExtensions := env.Get(EnvCompressExtensions, strings.Join(cfg.Extensions, ","))
+	compressMimeTypes := env.Get(EnvCompressMimeTypes, strings.Join(cfg.MimeTypes, ","))
+	if compressExtensions != "" || compressMimeTypes != "" {
+		if compressExtensions != "" {
+			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, config.ValueSeparator))
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
+			}
+			cfg.Extensions = extensions
+		}
+		if compressMimeTypes != "" {
+			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, config.ValueSeparator))
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", err, contenttypes)
+			}
+			cfg.MimeTypes = contenttypes
+		}
+	}
+
+	return cfg, nil
+}

cmd/config/constants.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,39 +14,22 @@
  * limitations under the License.
  */
 
-package disk
+package config
 
-import (
-	"os"
+// Config value separator
+const (
+	ValueSeparator = ","
 )
 
-func readFile(path string, buf []byte) (int, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return 0, err
-	}
-	defer f.Close()
+// Top level common ENVs
+const (
+	EnvAccessKey = "MINIO_ACCESS_KEY"
+	EnvSecretKey = "MINIO_SECRET_KEY"
+	EnvBrowser   = "MINIO_BROWSER"
+	EnvDomain    = "MINIO_DOMAIN"
+	EnvPublicIPs = "MINIO_PUBLIC_IPS"
+	EnvEndpoints = "MINIO_ENDPOINTS"
 
-	n, err := f.Read(buf)
-	if err != nil {
-		return 0, err
-	}
-
-	return n, nil
-}
-
-func writeFile(path string, data []byte) (int, error) {
-	f, err := os.Create(path)
-	if err != nil {
-		return 0, err
-	}
-	defer f.Close()
-
-	n, err := f.Write(data)
-	if err != nil {
-		return 0, err
-	}
-
-	f.Sync()
-	return n, nil
-}
+	EnvUpdate = "MINIO_UPDATE"
+	EnvWorm   = "MINIO_WORM"
+)

cmd/config/errors-utls.go

@@ -14,20 +14,22 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
+	"errors"
 	"fmt"
 	"io"
-	"net"
 	"os"
 	"syscall"
+
+	"github.com/minio/minio/pkg/color"
 )
 
-// uiErr is a structure which contains all information
+// Err is a structure which contains all information
 // to print a fatal error message in json or pretty mode
-// uiErr implements error so we can use it anywhere
-type uiErr struct {
+// Err implements error so we can use it anywhere
+type Err struct {
 	msg    string
 	detail string
 	action string
@@ -35,16 +37,16 @@ type uiErr struct {
 }
 
 // Return the error message
-func (u uiErr) Error() string {
+func (u Err) Error() string {
 	if u.detail == "" {
 		return u.msg
 	}
 	return u.detail
 }
 
-// Replace the current error's message
-func (u uiErr) Msg(m string, args ...interface{}) uiErr {
-	return uiErr{
+// Msg - Replace the current error's message
+func (u Err) Msg(m string, args ...interface{}) Err {
+	return Err{
 		msg:    fmt.Sprintf(m, args...),
 		detail: u.detail,
 		action: u.action,
@@ -52,14 +54,15 @@ func (u uiErr) Msg(m string, args ...interface{}) uiErr {
 	}
 }
 
-type uiErrFn func(err error) uiErr
+// ErrFn function wrapper
+type ErrFn func(err error) Err
 
 // Create a UI error generator, this is needed to simplify
 // the update of the detailed error message in several places
 // in MinIO code
-func newUIErrFn(msg, action, hint string) uiErrFn {
-	return func(err error) uiErr {
-		u := uiErr{
+func newErrFn(msg, action, hint string) ErrFn {
+	return func(err error) Err {
+		u := Err{
 			msg:    msg,
 			action: action,
 			hint:   hint,
@@ -71,47 +74,35 @@ func newUIErrFn(msg, action, hint string) uiErrFn {
 	}
 }
 
-// errorToUIError inspects the passed error and transforms it
+// ErrorToErr inspects the passed error and transforms it
 // to the appropriate UI error.
-func errorToUIErr(err error) uiErr {
-	// If this is already a uiErr, do nothing
-	if e, ok := err.(uiErr); ok {
+func ErrorToErr(err error) Err {
+	// If this is already a Err, do nothing
+	if e, ok := err.(Err); ok {
 		return e
 	}
 
 	// Show a generic message for known golang errors
-	switch e := err.(type) {
-	case *net.OpError:
-		if e.Op == "listen" {
-			if oe, ok := e.Err.(*os.SyscallError); ok {
-				if oe.Err == syscall.EADDRINUSE {
-					return uiErrPortAlreadyInUse(e).Msg("Specified port '" + e.Addr.String() + "' is already in use")
-				} else if oe.Err == syscall.EACCES {
-					return uiErrPortAccess(e).Msg("Insufficient permissions to use specified port '" + e.Addr.String() + "'")
-				}
-			}
-		}
-	case *os.PathError:
-		if os.IsPermission(e) {
-			return uiErrNoPermissionsToAccessDirFiles(e).Msg("Insufficient permissions to access path, `" + e.Path + "`")
-		}
-	}
-
-	switch err {
-	case io.ErrUnexpectedEOF:
-		return uiErrUnexpectedDataContent(err)
-	default:
+	if errors.Is(err, syscall.EADDRINUSE) {
+		return ErrPortAlreadyInUse(err).Msg("Specified port is already in use")
+	} else if errors.Is(err, syscall.EACCES) {
+		return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
+	} else if os.IsPermission(err) {
+		return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
+	} else if errors.Is(err, io.ErrUnexpectedEOF) {
+		return ErrUnexpectedDataContent(err)
+	} else {
 		// Failed to identify what type of error this, return a simple UI error
-		return uiErr{msg: err.Error()}
+		return Err{msg: err.Error()}
 	}
 
 }
 
-// fmtError() converts a fatal error message to a more clear error
+// FmtError converts a fatal error message to a more clear error
 // using some colors
-func fmtError(introMsg string, err error, jsonFlag bool) string {
+func FmtError(introMsg string, err error, jsonFlag bool) string {
 	renderedTxt := ""
-	uiErr := errorToUIErr(err)
+	uiErr := ErrorToErr(err)
 	// JSON print
 	if jsonFlag {
 		// Message text in json should be simple
@@ -123,18 +114,18 @@ func fmtError(introMsg string, err error, jsonFlag bool) string {
 	// Pretty print error message
 	introMsg += ": "
 	if uiErr.msg != "" {
-		introMsg += colorBold(uiErr.msg)
+		introMsg += color.Bold(uiErr.msg)
 	} else {
-		introMsg += colorBold(err.Error())
+		introMsg += color.Bold(err.Error())
 	}
-	renderedTxt += colorRed(introMsg) + "\n"
+	renderedTxt += color.Red(introMsg) + "\n"
 	// Add action message
 	if uiErr.action != "" {
-		renderedTxt += "> " + colorBgYellow(colorBlack(uiErr.action)) + "\n"
+		renderedTxt += "> " + color.BgYellow(color.Black(uiErr.action)) + "\n"
 	}
 	// Add hint
 	if uiErr.hint != "" {
-		renderedTxt += colorBold("HINT:") + "\n"
+		renderedTxt += color.Bold("HINT:") + "\n"
 		renderedTxt += "  " + uiErr.hint
 	}
 	return renderedTxt

cmd/config/errors.go

@@ -14,95 +14,102 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
+// UI errors
 var (
-	uiErrInvalidConfig = newUIErrFn(
+	ErrInvalidConfig = newErrFn(
 		"Invalid value found in the configuration file",
 		"Please ensure a valid value in the configuration file",
 		"For more details, refer to https://docs.min.io/docs/minio-server-configuration-guide",
 	)
 
-	uiErrInvalidBrowserValue = newUIErrFn(
+	ErrInvalidBrowserValue = newErrFn(
 		"Invalid browser value",
 		"Please check the passed value",
 		"Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`",
 	)
 
-	uiErrInvalidDomainValue = newUIErrFn(
+	ErrInvalidDomainValue = newErrFn(
 		"Invalid domain value",
 		"Please check the passed value",
 		"Domain can only accept DNS compatible values",
 	)
 
-	uiErrInvalidErasureSetSize = newUIErrFn(
+	ErrInvalidErasureSetSize = newErrFn(
 		"Invalid erasure set size",
 		"Please check the passed value",
 		"Erasure set can only accept any of [4, 6, 8, 10, 12, 14, 16] values",
 	)
 
-	uiErrInvalidWormValue = newUIErrFn(
+	ErrInvalidWormValue = newErrFn(
 		"Invalid WORM value",
 		"Please check the passed value",
 		"WORM can only accept `on` and `off` values. To enable WORM, set this value to `on`",
 	)
 
-	uiErrInvalidCacheDrivesValue = newUIErrFn(
+	ErrInvalidCacheDrivesValue = newErrFn(
 		"Invalid cache drive value",
 		"Please check the value in this ENV variable",
 		"MINIO_CACHE_DRIVES: Mounted drives or directories are delimited by `;`",
 	)
 
-	uiErrInvalidCacheExcludesValue = newUIErrFn(
+	ErrInvalidCacheExcludesValue = newErrFn(
 		"Invalid cache excludes value",
 		"Please check the passed value",
 		"MINIO_CACHE_EXCLUDE: Cache exclusion patterns are delimited by `;`",
 	)
 
-	uiErrInvalidCacheExpiryValue = newUIErrFn(
+	ErrInvalidCacheExpiryValue = newErrFn(
 		"Invalid cache expiry value",
 		"Please check the passed value",
 		"MINIO_CACHE_EXPIRY: Valid cache expiry duration is in days",
 	)
 
-	uiErrInvalidCacheMaxUse = newUIErrFn(
+	ErrInvalidCacheMaxUse = newErrFn(
 		"Invalid cache max-use value",
 		"Please check the passed value",
 		"MINIO_CACHE_MAXUSE: Valid cache max-use value between 0-100",
 	)
 
-	uiErrInvalidCredentials = newUIErrFn(
+	ErrInvalidCacheEncryptionKey = newErrFn(
+		"Invalid cache encryption master key value",
+		"Please check the passed value",
+		"MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide",
+	)
+
+	ErrInvalidCredentials = newErrFn(
 		"Invalid credentials",
 		"Please provide correct credentials",
 		`Access key length should be between minimum 3 characters in length.
 Secret key should be in between 8 and 40 characters`,
 	)
 
-	uiErrEnvCredentialsMissingGateway = newUIErrFn(
+	ErrEnvCredentialsMissingGateway = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
 		`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
 	)
 
-	uiErrEnvCredentialsMissingDistributed = newUIErrFn(
+	ErrEnvCredentialsMissingDistributed = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
 		`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
 	)
 
-	uiErrInvalidErasureEndpoints = newUIErrFn(
+	ErrInvalidErasureEndpoints = newErrFn(
 		"Invalid endpoint(s) in erasure mode",
 		"Please provide correct combination of local/remote paths",
 		"For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide",
 	)
 
-	uiErrInvalidNumberOfErasureEndpoints = newUIErrFn(
+	ErrInvalidNumberOfErasureEndpoints = newErrFn(
 		"Invalid total number of endpoints for erasure mode",
 		"Please provide an even number of endpoints greater or equal to 4",
 		"For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide",
 	)
 
-	uiErrStorageClassValue = newUIErrFn(
+	ErrStorageClassValue = newErrFn(
 		"Invalid storage class value",
 		"Please check the value",
 		`MINIO_STORAGE_CLASS_STANDARD: Format "EC:<Default_Parity_Standard_Class>" (e.g. "EC:3"). This sets the number of parity disks for MinIO server in Standard mode. Objects are stored in Standard mode, if storage class is not defined in Put request
@@ -110,13 +117,13 @@ MINIO_STORAGE_CLASS_RRS: Format "EC:<Default_Parity_Reduced_Redundancy_Class>" (
 Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storage-class for more information`,
 	)
 
-	uiErrUnexpectedBackendVersion = newUIErrFn(
+	ErrUnexpectedBackendVersion = newErrFn(
 		"Backend version seems to be too recent",
 		"Please update to the latest MinIO version",
 		"",
 	)
 
-	uiErrInvalidAddressFlag = newUIErrFn(
+	ErrInvalidAddressFlag = newErrFn(
 		"--address input is invalid",
 		"Please check --address parameter",
 		`--address binds to a specific ADDRESS:PORT, ADDRESS can be an IPv4/IPv6 address or hostname (default port is ':9000')
@@ -125,7 +132,7 @@ Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storag
 		  --address '[fe80::da00:a6c8:e3ae:ddd7]:9000'`,
 	)
 
-	uiErrInvalidFSEndpoint = newUIErrFn(
+	ErrInvalidFSEndpoint = newErrFn(
 		"Invalid endpoint for standalone FS mode",
 		"Please check the FS endpoint",
 		`FS mode requires only one writable disk path
@@ -133,91 +140,91 @@ Example 1:
    $ minio server /data/minio/`,
 	)
 
-	uiErrUnableToWriteInBackend = newUIErrFn(
+	ErrUnableToWriteInBackend = newErrFn(
 		"Unable to write to the backend",
 		"Please ensure MinIO binary has write permissions for the backend",
 		`Verify if MinIO binary is running as the same user who has write permissions for the backend`,
 	)
 
-	uiErrPortAlreadyInUse = newUIErrFn(
+	ErrPortAlreadyInUse = newErrFn(
 		"Port is already in use",
 		"Please ensure no other program uses the same address/port",
 		"",
 	)
 
-	uiErrPortAccess = newUIErrFn(
+	ErrPortAccess = newErrFn(
 		"Unable to use specified port",
 		"Please ensure MinIO binary has 'cap_net_bind_service=+ep' permissions",
 		`Use 'sudo setcap cap_net_bind_service=+ep /path/to/minio' to provide sufficient permissions`,
 	)
 
-	uiErrNoPermissionsToAccessDirFiles = newUIErrFn(
+	ErrNoPermissionsToAccessDirFiles = newErrFn(
 		"Missing permissions to access the specified path",
 		"Please ensure the specified path can be accessed",
 		"",
 	)
 
-	uiErrSSLUnexpectedError = newUIErrFn(
+	ErrSSLUnexpectedError = newErrFn(
 		"Invalid TLS certificate",
 		"Please check the content of your certificate data",
 		`Only PEM (x.509) format is accepted as valid public & private certificates`,
 	)
 
-	uiErrSSLUnexpectedData = newUIErrFn(
+	ErrSSLUnexpectedData = newErrFn(
 		"Invalid TLS certificate",
 		"Please check your certificate",
 		"",
 	)
 
-	uiErrSSLNoPassword = newUIErrFn(
+	ErrSSLNoPassword = newErrFn(
 		"Missing TLS password",
-		"Please set the password to environment variable `"+TLSPrivateKeyPassword+"` so that the private key can be decrypted",
+		"Please set the password to environment variable `MINIO_CERT_PASSWD` so that the private key can be decrypted",
 		"",
 	)
 
-	uiErrNoCertsAndHTTPSEndpoints = newUIErrFn(
+	ErrNoCertsAndHTTPSEndpoints = newErrFn(
 		"HTTPS specified in endpoints, but no TLS certificate is found on the local machine",
 		"Please add TLS certificate or use HTTP endpoints only",
 		"Refer to https://docs.min.io/docs/how-to-secure-access-to-minio-server-with-tls for information about how to load a TLS certificate in your server",
 	)
 
-	uiErrCertsAndHTTPEndpoints = newUIErrFn(
+	ErrCertsAndHTTPEndpoints = newErrFn(
 		"HTTP specified in endpoints, but the server in the local machine is configured with a TLS certificate",
 		"Please remove the certificate in the configuration directory or switch to HTTPS",
 		"",
 	)
 
-	uiErrSSLWrongPassword = newUIErrFn(
+	ErrSSLWrongPassword = newErrFn(
 		"Unable to decrypt the private key using the provided password",
-		"Please set the correct password in environment variable "+TLSPrivateKeyPassword,
+		"Please set the correct password in environment variable `MINIO_CERT_PASSWD`",
 		"",
 	)
 
-	uiErrUnexpectedDataContent = newUIErrFn(
+	ErrUnexpectedDataContent = newErrFn(
 		"Unexpected data content",
 		"Please contact MinIO at https://slack.min.io",
 		"",
 	)
 
-	uiErrUnexpectedError = newUIErrFn(
+	ErrUnexpectedError = newErrFn(
 		"Unexpected error",
 		"Please contact MinIO at https://slack.min.io",
 		"",
 	)
 
-	uiErrInvalidCompressionIncludesValue = newUIErrFn(
+	ErrInvalidCompressionIncludesValue = newErrFn(
 		"Invalid compression include value",
 		"Please check the passed value",
 		"Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_ATTR=\"A,B,C\"",
 	)
 
-	uiErrInvalidGWSSEValue = newUIErrFn(
+	ErrInvalidGWSSEValue = newErrFn(
 		"Invalid gateway SSE value",
 		"Please check the passed value",
 		"MINIO_GATEWAY_SSE: Gateway SSE accepts only C and S3 as valid values. Delimit by `;` to set more than one value",
 	)
 
-	uiErrInvalidGWSSEEnvValue = newUIErrFn(
+	ErrInvalidGWSSEEnvValue = newErrFn(
 		"Invalid gateway SSE configuration",
 		"",
 		"Refer to https://docs.min.io/docs/minio-kms-quickstart-guide.html for setting up SSE",

cmd/config/etcd/etcd.go

@@ -0,0 +1,96 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package etcd
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"strings"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+const (
+	// Default values used while communicating with etcd.
+	defaultDialTimeout   = 30 * time.Second
+	defaultDialKeepAlive = 30 * time.Second
+)
+
+// etcd environment values
+const (
+	EnvEtcdEndpoints     = "MINIO_ETCD_ENDPOINTS"
+	EnvEtcdClientCert    = "MINIO_ETCD_CLIENT_CERT"
+	EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY"
+)
+
+// New - Initialize new etcd client
+func New(rootCAs *x509.CertPool) (*clientv3.Client, error) {
+	envEndpoints := env.Get(EnvEtcdEndpoints, "")
+	if envEndpoints == "" {
+		// etcd is not configured, nothing to do.
+		return nil, nil
+	}
+
+	etcdEndpoints := strings.Split(envEndpoints, config.ValueSeparator)
+
+	var etcdSecure bool
+	for _, endpoint := range etcdEndpoints {
+		u, err := xnet.ParseURL(endpoint)
+		if err != nil {
+			return nil, err
+		}
+		// If one of the endpoint is https, we will use https directly.
+		etcdSecure = etcdSecure || u.Scheme == "https"
+	}
+
+	var err error
+	var etcdClnt *clientv3.Client
+	if etcdSecure {
+		// This is only to support client side certificate authentication
+		// https://coreos.com/etcd/docs/latest/op-guide/security.html
+		etcdClientCertFile, ok1 := env.Lookup(EnvEtcdClientCert)
+		etcdClientCertKey, ok2 := env.Lookup(EnvEtcdClientCertKey)
+		var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+		if ok1 && ok2 {
+			getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+				cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
+				return &cert, terr
+			}
+		}
+
+		etcdClnt, err = clientv3.New(clientv3.Config{
+			Endpoints:         etcdEndpoints,
+			DialTimeout:       defaultDialTimeout,
+			DialKeepAliveTime: defaultDialKeepAlive,
+			TLS: &tls.Config{
+				RootCAs:              rootCAs,
+				GetClientCertificate: getClientCertificate,
+			},
+		})
+	} else {
+		etcdClnt, err = clientv3.New(clientv3.Config{
+			Endpoints:         etcdEndpoints,
+			DialTimeout:       defaultDialTimeout,
+			DialKeepAliveTime: defaultDialKeepAlive,
+		})
+	}
+	return etcdClnt, err
+}

cmd/config/ldap/config.go

@@ -0,0 +1,201 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ldap
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/minio/minio/pkg/env"
+	ldap "gopkg.in/ldap.v3"
+)
+
+const (
+	defaultLDAPExpiry = time.Hour * 1
+)
+
+// Config contains AD/LDAP server connectivity information.
+type Config struct {
+	IsEnabled bool `json:"enabled"`
+
+	// E.g. "ldap.minio.io:636"
+	ServerAddr string `json:"serverAddr"`
+
+	// STS credentials expiry duration
+	STSExpiryDuration string        `json:"stsExpiryDuration"`
+	stsExpiryDuration time.Duration // contains converted value
+
+	RootCAs *x509.CertPool `json:"-"`
+
+	// Format string for usernames
+	UsernameFormat string `json:"usernameFormat"`
+
+	GroupSearchBaseDN  string `json:"groupSearchBaseDN"`
+	GroupSearchFilter  string `json:"groupSearchFilter"`
+	GroupNameAttribute string `json:"groupNameAttribute"`
+}
+
+// LDAP keys and envs.
+const (
+	ServerAddr         = "server_addr"
+	STSExpiry          = "sts_expiry"
+	UsernameFormat     = "username_format"
+	GroupSearchFilter  = "group_search_filter"
+	GroupNameAttribute = "group_name_attribute"
+	GroupSearchBaseDN  = "group_search_base_dn"
+
+	EnvServerAddr         = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
+	EnvSTSExpiry          = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
+	EnvUsernameFormat     = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
+	EnvGroupSearchFilter  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
+	EnvGroupNameAttribute = "MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE"
+	EnvGroupSearchBaseDN  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
+)
+
+// Connect connect to ldap server.
+func (l *Config) Connect() (ldapConn *ldap.Conn, err error) {
+	if l == nil {
+		// Happens when LDAP is not configured.
+		return
+	}
+	return ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{RootCAs: l.RootCAs})
+}
+
+// GetExpiryDuration - return parsed expiry duration.
+func (l Config) GetExpiryDuration() time.Duration {
+	return l.stsExpiryDuration
+}
+
+// Lookup - initializes LDAP config, overrides config, if any ENV values are set.
+func Lookup(cfg Config, rootCAs *x509.CertPool) (l Config, err error) {
+	if cfg.ServerAddr == "" && cfg.IsEnabled {
+		return l, errors.New("ldap server cannot initialize with empty LDAP server")
+	}
+	l.RootCAs = rootCAs
+	ldapServer := env.Get(EnvServerAddr, cfg.ServerAddr)
+	if ldapServer == "" {
+		return l, nil
+	}
+	l.IsEnabled = true
+	l.ServerAddr = ldapServer
+	l.stsExpiryDuration = defaultLDAPExpiry
+	if v := env.Get(EnvSTSExpiry, cfg.STSExpiryDuration); v != "" {
+		expDur, err := time.ParseDuration(v)
+		if err != nil {
+			return l, errors.New("LDAP expiry time err:" + err.Error())
+		}
+		if expDur <= 0 {
+			return l, errors.New("LDAP expiry time has to be positive")
+		}
+		l.STSExpiryDuration = v
+		l.stsExpiryDuration = expDur
+	}
+
+	if v := env.Get(EnvUsernameFormat, cfg.UsernameFormat); v != "" {
+		subs, err := NewSubstituter("username", "test")
+		if err != nil {
+			return l, err
+		}
+		if _, err := subs.Substitute(v); err != nil {
+			return l, fmt.Errorf("Only username may be substituted in the username format: %s", err)
+		}
+		l.UsernameFormat = v
+	}
+
+	grpSearchFilter := env.Get(EnvGroupSearchFilter, cfg.GroupSearchFilter)
+	grpSearchNameAttr := env.Get(EnvGroupNameAttribute, cfg.GroupNameAttribute)
+	grpSearchBaseDN := env.Get(EnvGroupSearchBaseDN, cfg.GroupSearchBaseDN)
+
+	// Either all group params must be set or none must be set.
+	allNotSet := grpSearchFilter == "" && grpSearchNameAttr == "" && grpSearchBaseDN == ""
+	allSet := grpSearchFilter != "" && grpSearchNameAttr != "" && grpSearchBaseDN != ""
+	if !allNotSet && !allSet {
+		return l, errors.New("All group related parameters must be set")
+	}
+
+	if allSet {
+		subs, err := NewSubstituter("username", "test", "usernamedn", "test2")
+		if err != nil {
+			return l, err
+		}
+		if _, err := subs.Substitute(grpSearchFilter); err != nil {
+			return l, fmt.Errorf("Only username and usernamedn may be substituted in the group search filter string: %s", err)
+		}
+		l.GroupSearchFilter = grpSearchFilter
+		l.GroupNameAttribute = grpSearchNameAttr
+		subs, err = NewSubstituter("username", "test", "usernamedn", "test2")
+		if err != nil {
+			return l, err
+		}
+		if _, err := subs.Substitute(grpSearchBaseDN); err != nil {
+			return l, fmt.Errorf("Only username and usernamedn may be substituted in the base DN string: %s", err)
+		}
+		l.GroupSearchBaseDN = grpSearchBaseDN
+	}
+	return
+}
+
+// Substituter - This type is to allow restricted runtime
+// substitutions of variables in LDAP configuration items during
+// runtime.
+type Substituter struct {
+	vals map[string]string
+}
+
+// NewSubstituter - sets up the substituter for usage, for e.g.:
+//
+//  subber := NewSubstituter("username", "john")
+func NewSubstituter(v ...string) (Substituter, error) {
+	if len(v)%2 != 0 {
+		return Substituter{}, errors.New("Need an even number of arguments")
+	}
+	vals := make(map[string]string)
+	for i := 0; i < len(v); i += 2 {
+		vals[v[i]] = v[i+1]
+	}
+	return Substituter{vals: vals}, nil
+}
+
+// Substitute - performs substitution on the given string `t`. Returns
+// an error if there are any variables in the input that do not have
+// values in the substituter. E.g.:
+//
+// subber.Substitute("uid=${username},cn=users,dc=example,dc=com")
+//
+// returns "uid=john,cn=users,dc=example,dc=com"
+//
+// whereas:
+//
+// subber.Substitute("uid=${usernamedn}")
+//
+// returns an error.
+func (s *Substituter) Substitute(t string) (string, error) {
+	for k, v := range s.vals {
+		re := regexp.MustCompile(fmt.Sprintf(`\$\{%s\}`, k))
+		t = re.ReplaceAllLiteralString(t, v)
+	}
+	// Check if all requested substitutions have been made.
+	re := regexp.MustCompile(`\$\{.*\}`)
+	if re.MatchString(t) {
+		return "", errors.New("unsupported substitution requested")
+	}
+	return t, nil
+}

cmd/config/ldap/config_test.go

@@ -0,0 +1,65 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ldap
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestSubstituter(t *testing.T) {
+	tests := []struct {
+		KV               []string
+		SubstitutableStr string
+		SubstitutedStr   string
+		ErrExpected      bool
+	}{
+		{
+			KV:               []string{"username", "john"},
+			SubstitutableStr: "uid=${username},cn=users,dc=example,dc=com",
+			SubstitutedStr:   "uid=john,cn=users,dc=example,dc=com",
+			ErrExpected:      false,
+		},
+		{
+			KV:               []string{"username", "john"},
+			SubstitutableStr: "uid=${usernamedn},cn=users,dc=example,dc=com",
+			ErrExpected:      true,
+		},
+		{
+			KV:               []string{"username"},
+			SubstitutableStr: "uid=${usernamedn},cn=users,dc=example,dc=com",
+			ErrExpected:      true,
+		},
+	}
+
+	for i, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
+			subber, err := NewSubstituter(test.KV...)
+			if err != nil && !test.ErrExpected {
+				t.Errorf("Unexpected failure %s", err)
+			}
+			gotStr, err := subber.Substitute(test.SubstitutableStr)
+			if err != nil && !test.ErrExpected {
+				t.Errorf("Unexpected failure %s", err)
+			}
+			if gotStr != test.SubstitutedStr {
+				t.Errorf("Expected %s, got %s", test.SubstitutedStr, gotStr)
+			}
+		})
+	}
+}

cmd/config/notify/config.go

@@ -0,0 +1,66 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package notify
+
+import "github.com/minio/minio/pkg/event/target"
+
+// Config - notification target configuration structure, holds
+// information about various notification targets.
+type Config struct {
+	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
+	Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
+	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
+	MQTT          map[string]target.MQTTArgs          `json:"mqtt"`
+	MySQL         map[string]target.MySQLArgs         `json:"mysql"`
+	NATS          map[string]target.NATSArgs          `json:"nats"`
+	NSQ           map[string]target.NSQArgs           `json:"nsq"`
+	PostgreSQL    map[string]target.PostgreSQLArgs    `json:"postgresql"`
+	Redis         map[string]target.RedisArgs         `json:"redis"`
+	Webhook       map[string]target.WebhookArgs       `json:"webhook"`
+}
+
+const (
+	defaultTarget = "1"
+)
+
+// NewConfig - initialize notification config.
+func NewConfig() Config {
+	// Make sure to initialize notification targets
+	cfg := Config{
+		NSQ:           make(map[string]target.NSQArgs),
+		AMQP:          make(map[string]target.AMQPArgs),
+		MQTT:          make(map[string]target.MQTTArgs),
+		NATS:          make(map[string]target.NATSArgs),
+		Redis:         make(map[string]target.RedisArgs),
+		MySQL:         make(map[string]target.MySQLArgs),
+		Kafka:         make(map[string]target.KafkaArgs),
+		Webhook:       make(map[string]target.WebhookArgs),
+		PostgreSQL:    make(map[string]target.PostgreSQLArgs),
+		Elasticsearch: make(map[string]target.ElasticsearchArgs),
+	}
+	cfg.NSQ[defaultTarget] = target.NSQArgs{}
+	cfg.AMQP[defaultTarget] = target.AMQPArgs{}
+	cfg.MQTT[defaultTarget] = target.MQTTArgs{}
+	cfg.NATS[defaultTarget] = target.NATSArgs{}
+	cfg.Redis[defaultTarget] = target.RedisArgs{}
+	cfg.MySQL[defaultTarget] = target.MySQLArgs{}
+	cfg.Kafka[defaultTarget] = target.KafkaArgs{}
+	cfg.Webhook[defaultTarget] = target.WebhookArgs{}
+	cfg.PostgreSQL[defaultTarget] = target.PostgreSQLArgs{}
+	cfg.Elasticsearch[defaultTarget] = target.ElasticsearchArgs{}
+	return cfg
+}

cmd/config/storageclass/storage-class.go

@@ -0,0 +1,230 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package storageclass
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Standard constants for all storage class
+const (
+	// Reduced redundancy storage class
+	RRS = "REDUCED_REDUNDANCY"
+	// Standard storage class
+	STANDARD = "STANDARD"
+)
+
+// Standard constats for config info storage class
+const (
+	// Reduced redundancy storage class environment variable
+	RRSEnv = "MINIO_STORAGE_CLASS_RRS"
+	// Standard storage class environment variable
+	StandardEnv = "MINIO_STORAGE_CLASS_STANDARD"
+
+	// Supported storage class scheme is EC
+	schemePrefix = "EC"
+
+	// Min parity disks
+	minParityDisks = 2
+
+	// Default RRS parity is always minimum parity.
+	defaultRRSParity = minParityDisks
+)
+
+// StorageClass - holds storage class information
+type StorageClass struct {
+	Parity int
+}
+
+// Config storage class configuration
+type Config struct {
+	Standard StorageClass `json:"standard"`
+	RRS      StorageClass `json:"rrs"`
+}
+
+// UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
+func (sCfg *Config) UnmarshalJSON(data []byte) error {
+	type Alias Config
+	aux := &struct {
+		*Alias
+	}{
+		Alias: (*Alias)(sCfg),
+	}
+	return json.Unmarshal(data, &aux)
+}
+
+// IsValid - returns true if input string is a valid
+// storage class kind supported.
+func IsValid(sc string) bool {
+	return sc == RRS || sc == STANDARD
+}
+
+// UnmarshalText unmarshals storage class from its textual form into
+// storageClass structure.
+func (sc *StorageClass) UnmarshalText(b []byte) error {
+	scStr := string(b)
+	if scStr == "" {
+		return nil
+	}
+	s, err := parseStorageClass(scStr)
+	if err != nil {
+		return err
+	}
+	sc.Parity = s.Parity
+	return nil
+}
+
+// MarshalText - marshals storage class string.
+func (sc *StorageClass) MarshalText() ([]byte, error) {
+	if sc.Parity != 0 {
+		return []byte(fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)), nil
+	}
+	return []byte(""), nil
+}
+
+func (sc *StorageClass) String() string {
+	if sc.Parity != 0 {
+		return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
+	}
+	return ""
+}
+
+// Parses given storageClassEnv and returns a storageClass structure.
+// Supported Storage Class format is "Scheme:Number of parity disks".
+// Currently only supported scheme is "EC".
+func parseStorageClass(storageClassEnv string) (sc StorageClass, err error) {
+	s := strings.Split(storageClassEnv, ":")
+
+	// only two elements allowed in the string - "scheme" and "number of parity disks"
+	if len(s) > 2 {
+		return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too many sections in " + storageClassEnv)
+	} else if len(s) < 2 {
+		return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too few sections in " + storageClassEnv)
+	}
+
+	// only allowed scheme is "EC"
+	if s[0] != schemePrefix {
+		return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Unsupported scheme " + s[0] + ". Supported scheme is EC")
+	}
+
+	// Number of parity disks should be integer
+	parityDisks, err := strconv.Atoi(s[1])
+	if err != nil {
+		return StorageClass{}, config.ErrStorageClassValue(err)
+	}
+
+	return StorageClass{
+		Parity: parityDisks,
+	}, nil
+}
+
+// Validates the parity disks.
+func validateParity(ssParity, rrsParity, drivesPerSet int) (err error) {
+	if ssParity == 0 && rrsParity == 0 {
+		return nil
+	}
+
+	// SS parity disks should be greater than or equal to minParityDisks.
+	// Parity below minParityDisks is not supported.
+	if ssParity < minParityDisks {
+		return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d",
+			ssParity, minParityDisks)
+	}
+
+	// RRS parity disks should be greater than or equal to minParityDisks.
+	// Parity below minParityDisks is not supported.
+	if rrsParity < minParityDisks {
+		return fmt.Errorf("Reduced redundancy storage class parity %d should be greater than or equal to %d", rrsParity, minParityDisks)
+	}
+
+	if ssParity > drivesPerSet/2 {
+		return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, drivesPerSet/2)
+	}
+
+	if rrsParity > drivesPerSet/2 {
+		return fmt.Errorf("Reduced redundancy storage class parity %d should be less than  or equal to %d", rrsParity, drivesPerSet/2)
+	}
+
+	if ssParity > 0 && rrsParity > 0 {
+		if ssParity < rrsParity {
+			return fmt.Errorf("Standard storage class parity disks %d should be greater than or equal to Reduced redundancy storage class parity disks %d", ssParity, rrsParity)
+		}
+	}
+	return nil
+}
+
+// GetParityForSC - Returns the data and parity drive count based on storage class
+// If storage class is set using the env vars MINIO_STORAGE_CLASS_RRS and MINIO_STORAGE_CLASS_STANDARD
+// or config.json fields
+// -- corresponding values are returned
+// If storage class is not set during startup, default values are returned
+// -- Default for Reduced Redundancy Storage class is, parity = 2 and data = N-Parity
+// -- Default for Standard Storage class is, parity = N/2, data = N/2
+// If storage class is empty
+// -- standard storage class is assumed and corresponding data and parity is returned
+func (sCfg Config) GetParityForSC(sc string) (parity int) {
+	switch strings.TrimSpace(sc) {
+	case RRS:
+		// set the rrs parity if available
+		if sCfg.RRS.Parity == 0 {
+			return defaultRRSParity
+		}
+		return sCfg.RRS.Parity
+	default:
+		return sCfg.Standard.Parity
+	}
+}
+
+// LookupConfig - lookup storage class config and override with valid environment settings if any.
+func LookupConfig(cfg Config, drivesPerSet int) (Config, error) {
+	var err error
+
+	// Check for environment variables and parse into storageClass struct
+	if ssc := env.Get(StandardEnv, cfg.Standard.String()); ssc != "" {
+		cfg.Standard, err = parseStorageClass(ssc)
+		if err != nil {
+			return cfg, err
+		}
+	}
+	if cfg.Standard.Parity == 0 {
+		cfg.Standard.Parity = drivesPerSet / 2
+	}
+
+	if rrsc := env.Get(RRSEnv, cfg.RRS.String()); rrsc != "" {
+		cfg.RRS, err = parseStorageClass(rrsc)
+		if err != nil {
+			return cfg, err
+		}
+	}
+	if cfg.RRS.Parity == 0 {
+		cfg.RRS.Parity = defaultRRSParity
+	}
+
+	// Validation is done after parsing both the storage classes. This is needed because we need one
+	// storage class value to deduce the correct value of the other storage class.
+	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, drivesPerSet); err != nil {
+		return cfg, err
+	}
+
+	return cfg, nil
+}

cmd/config/storageclass/storage-class_test.go

@@ -0,0 +1,163 @@
+/*
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package storageclass
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func TestParseStorageClass(t *testing.T) {
+	tests := []struct {
+		storageClassEnv string
+		wantSc          StorageClass
+		expectedError   error
+	}{
+		{"EC:3", StorageClass{
+			Parity: 3},
+			nil},
+		{"EC:4", StorageClass{
+			Parity: 4},
+			nil},
+		{"AB:4", StorageClass{
+			Parity: 4},
+			errors.New("Unsupported scheme AB. Supported scheme is EC")},
+		{"EC:4:5", StorageClass{
+			Parity: 4},
+			errors.New("Too many sections in EC:4:5")},
+		{"EC:A", StorageClass{
+			Parity: 4},
+			errors.New(`strconv.Atoi: parsing "A": invalid syntax`)},
+		{"AB", StorageClass{
+			Parity: 4},
+			errors.New("Too few sections in AB")},
+	}
+	for i, tt := range tests {
+		gotSc, err := parseStorageClass(tt.storageClassEnv)
+		if err != nil && tt.expectedError == nil {
+			t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
+			return
+		}
+		if err == nil && tt.expectedError != nil {
+			t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
+			return
+		}
+		if tt.expectedError == nil && !reflect.DeepEqual(gotSc, tt.wantSc) {
+			t.Errorf("Test %d, Expected %v, got %v", i+1, tt.wantSc, gotSc)
+			return
+		}
+		if tt.expectedError != nil && err.Error() != tt.expectedError.Error() {
+			t.Errorf("Test %d, Expected `%v`, got `%v`", i+1, tt.expectedError, err)
+		}
+	}
+}
+
+func TestValidateParity(t *testing.T) {
+	tests := []struct {
+		rrsParity    int
+		ssParity     int
+		success      bool
+		drivesPerSet int
+	}{
+		{2, 4, true, 16},
+		{3, 3, true, 16},
+		{0, 0, true, 16},
+		{1, 4, false, 16},
+		{7, 6, false, 16},
+		{9, 0, false, 16},
+		{9, 9, false, 16},
+		{2, 9, false, 16},
+		{9, 2, false, 16},
+	}
+	for i, tt := range tests {
+		err := validateParity(tt.ssParity, tt.rrsParity, tt.drivesPerSet)
+		if err != nil && tt.success {
+			t.Errorf("Test %d, Expected success, got %s", i+1, err)
+		}
+		if err == nil && !tt.success {
+			t.Errorf("Test %d, Expected failure, got success", i+1)
+		}
+	}
+}
+
+func TestParityCount(t *testing.T) {
+	tests := []struct {
+		sc             string
+		disksCount     int
+		expectedData   int
+		expectedParity int
+	}{
+		{RRS, 16, 14, 2},
+		{STANDARD, 16, 8, 8},
+		{"", 16, 8, 8},
+		{RRS, 16, 9, 7},
+		{STANDARD, 16, 10, 6},
+		{"", 16, 9, 7},
+	}
+	for i, tt := range tests {
+		scfg := Config{
+			Standard: StorageClass{
+				Parity: 8,
+			},
+			RRS: StorageClass{
+				Parity: 0,
+			},
+		}
+		// Set env var for test case 4
+		if i+1 == 4 {
+			scfg.RRS.Parity = 7
+		}
+		// Set env var for test case 5
+		if i+1 == 5 {
+			scfg.Standard.Parity = 6
+		}
+		// Set env var for test case 6
+		if i+1 == 6 {
+			scfg.Standard.Parity = 7
+		}
+		parity := scfg.GetParityForSC(tt.sc)
+		if (tt.disksCount - parity) != tt.expectedData {
+			t.Errorf("Test %d, Expected data disks %d, got %d", i+1, tt.expectedData, tt.disksCount-parity)
+			continue
+		}
+		if parity != tt.expectedParity {
+			t.Errorf("Test %d, Expected parity disks %d, got %d", i+1, tt.expectedParity, parity)
+		}
+	}
+}
+
+// Test IsValid method with valid and invalid inputs
+func TestIsValidStorageClassKind(t *testing.T) {
+	tests := []struct {
+		sc   string
+		want bool
+	}{
+		{"STANDARD", true},
+		{"REDUCED_REDUNDANCY", true},
+		{"", false},
+		{"INVALID", false},
+		{"123", false},
+		{"MINIO_STORAGE_CLASS_RRS", false},
+		{"MINIO_STORAGE_CLASS_STANDARD", false},
+	}
+	for i, tt := range tests {
+		if got := IsValid(tt.sc); got != tt.want {
+			t.Errorf("Test %d, Expected Storage Class to be %t, got %t", i+1, tt.want, got)
+		}
+	}
+}

cmd/consolelogger.go

@@ -0,0 +1,142 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	ring "container/ring"
+	"context"
+	"sync"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/message/log"
+	"github.com/minio/minio/cmd/logger/target/console"
+	"github.com/minio/minio/pkg/madmin"
+	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/pubsub"
+)
+
+// number of log messages to buffer
+const defaultLogBufferCount = 10000
+
+//HTTPConsoleLoggerSys holds global console logger state
+type HTTPConsoleLoggerSys struct {
+	pubsub   *pubsub.PubSub
+	console  *console.Target
+	nodeName string
+	// To protect ring buffer.
+	logBufLk sync.RWMutex
+	logBuf   *ring.Ring
+}
+
+// NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to
+// the console logging pub sub system
+func NewConsoleLogger(ctx context.Context, endpoints EndpointList) *HTTPConsoleLoggerSys {
+	host, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		logger.FatalIf(err, "Unable to start console logging subsystem")
+	}
+	var nodeName string
+	if globalIsDistXL {
+		nodeName = host.Name
+	}
+	ps := pubsub.New()
+	return &HTTPConsoleLoggerSys{
+		ps, nil, nodeName, sync.RWMutex{}, ring.New(defaultLogBufferCount),
+	}
+}
+
+// HasLogListeners returns true if console log listeners are registered
+// for this node or peers
+func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
+	return sys != nil && sys.pubsub.HasSubscribers()
+}
+
+// Subscribe starts console logging for this node.
+func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan struct{}, node string, last int, filter func(entry interface{}) bool) {
+	// Enable console logging for remote client even if local console logging is disabled in the config.
+	if !globalServerConfig.Logger.Console.Enabled && !sys.pubsub.HasSubscribers() {
+		logger.AddTarget(globalConsoleSys.Console())
+	}
+
+	cnt := 0
+	// by default send all console logs in the ring buffer unless node or limit query parameters
+	// are set.
+	var lastN []madmin.LogInfo
+	if last > defaultLogBufferCount || last <= 0 {
+		last = defaultLogBufferCount
+	}
+
+	lastN = make([]madmin.LogInfo, last)
+	sys.logBufLk.RLock()
+	sys.logBuf.Do(func(p interface{}) {
+		if p != nil && (p.(madmin.LogInfo)).SendLog(node) {
+			lastN[cnt%last] = p.(madmin.LogInfo)
+			cnt++
+		}
+	})
+	sys.logBufLk.RUnlock()
+	// send last n console log messages in order filtered by node
+	if cnt > 0 {
+		for i := 0; i < last; i++ {
+			entry := lastN[(cnt+i)%last]
+			if (entry == madmin.LogInfo{}) {
+				continue
+			}
+			select {
+			case subCh <- entry:
+			case <-doneCh:
+				return
+			}
+		}
+	}
+	sys.pubsub.Subscribe(subCh, doneCh, filter)
+}
+
+// Console returns a console target
+func (sys *HTTPConsoleLoggerSys) Console() *HTTPConsoleLoggerSys {
+	if sys == nil {
+		return sys
+	}
+	if sys.console == nil {
+		sys.console = console.New()
+	}
+	return sys
+}
+
+// Send log message 'e' to console and publish to console
+// log pubsub system
+func (sys *HTTPConsoleLoggerSys) Send(e interface{}) error {
+	var lg madmin.LogInfo
+	switch e := e.(type) {
+	case log.Entry:
+		lg = madmin.LogInfo{Entry: e, NodeName: sys.nodeName}
+	case string:
+		lg = madmin.LogInfo{ConsoleMsg: e, NodeName: sys.nodeName}
+	}
+
+	sys.pubsub.Publish(lg)
+	sys.logBufLk.Lock()
+	// add log to ring buffer
+	sys.logBuf.Value = lg
+	sys.logBuf = sys.logBuf.Next()
+	sys.logBufLk.Unlock()
+
+	if globalServerConfig.Logger.Console.Enabled {
+		return sys.console.Send(e)
+	}
+	return nil
+}

cmd/crypto/config.go

@@ -1,4 +1,4 @@
-// MinIO Cloud Storage, (C) 2015, 2016, 2017, 2018 MinIO, Inc.
+// MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,8 +14,129 @@
 
 package crypto
 
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/pkg/env"
+)
+
+const (
+	// EnvKMSMasterKey is the environment variable used to specify
+	// a KMS master key used to protect SSE-S3 per-object keys.
+	// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
+	EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY"
+
+	// EnvAutoEncryption is the environment variable used to en/disable
+	// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
+	// requires a valid KMS configuration and turns any non-SSE-C
+	// request into an SSE-S3 request.
+	// If present EnvAutoEncryption must be either "on" or "off".
+	EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION"
+)
+
+const (
+	// EnvVaultEndpoint is the environment variable used to specify
+	// the vault HTTPS endpoint.
+	EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
+
+	// EnvVaultAuthType is the environment variable used to specify
+	// the authentication type for vault.
+	EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
+
+	// EnvVaultAppRoleID is the environment variable used to specify
+	// the vault AppRole ID.
+	EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
+
+	// EnvVaultAppSecretID is the environment variable used to specify
+	// the vault AppRole secret corresponding to the AppRole ID.
+	EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
+
+	// EnvVaultKeyVersion is the environment variable used to specify
+	// the vault key version.
+	EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
+
+	// EnvVaultKeyName is the environment variable used to specify
+	// the vault named key-ring. In the S3 context it's referred as
+	// customer master key ID (CMK-ID).
+	EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
+
+	// EnvVaultCAPath is the environment variable used to specify the
+	// path to a directory of PEM-encoded CA cert files. These CA cert
+	// files are used to authenticate MinIO to Vault over mTLS.
+	EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
+
+	// EnvVaultNamespace is the environment variable used to specify
+	// vault namespace. The vault namespace is used if the enterprise
+	// version of Hashicorp Vault is used.
+	EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
+)
+
 // KMSConfig has the KMS config for hashicorp vault
 type KMSConfig struct {
 	AutoEncryption bool        `json:"-"`
 	Vault          VaultConfig `json:"vault"`
 }
+
+// LookupConfig extracts the KMS configuration provided by environment
+// variables and merge them with the provided KMS configuration. The
+// merging follows the following rules:
+//
+// 1. A valid value provided as environment variable is higher prioritized
+// than the provided configuration and overwrites the value from the
+// configuration file.
+//
+// 2. A value specified as environment variable never changes the configuration
+// file. So it is never made a persistent setting.
+//
+// It sets the global KMS configuration according to the merged configuration
+// on succes.
+func LookupConfig(config KMSConfig) (KMSConfig, error) {
+	var err error
+	// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
+	config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint)
+	config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath)
+	config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type)
+	config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID)
+	config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret)
+	config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name)
+	config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace)
+	keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version))
+	config.Vault.Key.Version, err = strconv.Atoi(keyVersion)
+	if err != nil {
+		return config, fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion)
+	}
+	if err = config.Vault.Verify(); err != nil {
+		return config, err
+	}
+
+	return config, nil
+}
+
+// NewKMS - initialize a new KMS.
+func NewKMS(config KMSConfig) (kms KMS, err error) {
+	// Lookup KMS master keys - only available through ENV.
+	if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
+		if !config.Vault.IsEmpty() { // Vault and KMS master key provided
+			return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
+		}
+		kms, err = ParseMasterKey(masterKey)
+		if err != nil {
+			return kms, err
+		}
+	}
+	if !config.Vault.IsEmpty() {
+		kms, err = NewVault(config.Vault)
+		if err != nil {
+			return kms, err
+		}
+	}
+
+	autoEncryption := strings.EqualFold(env.Get(EnvAutoEncryption, "off"), "on")
+	if autoEncryption && kms == nil {
+		return kms, errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present")
+	}
+	return kms, nil
+}

cmd/crypto/error.go

@@ -19,9 +19,9 @@ import "errors"
 // Error is the generic type for any error happening during decrypting
 // an object. It indicates that the object itself or its metadata was
 // modified accidentally or maliciously.
-type Error struct{ msg string }
+type Error string
 
-func (e Error) Error() string { return e.msg }
+func (e Error) Error() string { return string(e) }
 
 var (
 	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
@@ -56,14 +56,14 @@ var (
 	ErrIncompatibleEncryptionMethod = errors.New("Server side encryption specified with both SSE-C and SSE-S3 headers")
 )
 
-var (
-	errMissingInternalIV            = Error{"The object metadata is missing the internal encryption IV"}
-	errMissingInternalSealAlgorithm = Error{"The object metadata is missing the internal seal algorithm"}
+const (
+	errMissingInternalIV            Error = "The object metadata is missing the internal encryption IV"
+	errMissingInternalSealAlgorithm Error = "The object metadata is missing the internal seal algorithm"
 
-	errInvalidInternalIV            = Error{"The internal encryption IV is malformed"}
-	errInvalidInternalSealAlgorithm = Error{"The internal seal algorithm is invalid and not supported"}
+	errInvalidInternalIV            Error = "The internal encryption IV is malformed"
+	errInvalidInternalSealAlgorithm Error = "The internal seal algorithm is invalid and not supported"
 
-	errMissingUpdatedKey = Error{"The key update returned no error but also no sealed key"}
+	errMissingUpdatedKey Error = "The key update returned no error but also no sealed key"
 )
 
 var (

cmd/crypto/header.go

@@ -83,6 +83,13 @@ func RemoveSensitiveHeaders(h http.Header) {
 	h.Del(SSECopyKey)
 }
 
+// IsRequested returns true if the HTTP headers indicates
+// that any form server-side encryption (SSE-C, SSE-S3 or SSE-KMS)
+// is requested.
+func IsRequested(h http.Header) bool {
+	return S3.IsRequested(h) || SSEC.IsRequested(h) || SSECopy.IsRequested(h) || S3KMS.IsRequested(h)
+}
+
 // S3 represents AWS SSE-S3. It provides functionality to handle
 // SSE-S3 requests.
 var S3 = s3{}

cmd/crypto/header_test.go

@@ -20,6 +20,29 @@ import (
 	"testing"
 )
 
+func TestIsRequested(t *testing.T) {
+	for i, test := range kmsIsRequestedTests {
+		if got := IsRequested(test.Header) && S3KMS.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("SSE-KMS: Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+	for i, test := range s3IsRequestedTests {
+		if got := IsRequested(test.Header) && S3.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("SSE-S3: Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+	for i, test := range ssecIsRequestedTests {
+		if got := IsRequested(test.Header) && SSEC.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+	for i, test := range ssecCopyIsRequestedTests {
+		if got := IsRequested(test.Header) && SSECopy.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
 var kmsIsRequestedTests = []struct {
 	Header   http.Header
 	Expected bool

cmd/crypto/key.go

@@ -108,7 +108,7 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 	)
 	switch sealedKey.Algorithm {
 	default:
-		return Error{fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm)}
+		return Error(fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm))
 	case SealAlgorithm:
 		mac := hmac.New(sha256.New, extKey[:])
 		mac.Write(sealedKey.IV[:])

cmd/crypto/kms.go

@@ -72,6 +72,9 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 // data key generation and unsealing of KMS-generated
 // data keys.
 type KMS interface {
+	// KeyID - returns configured KMS key id.
+	KeyID() string
+
 	// GenerateKey generates a new random data key using
 	// the master key referenced by the keyID. It returns
 	// the plaintext key and the sealed plaintext key
@@ -102,14 +105,19 @@ type KMS interface {
 }
 
 type masterKeyKMS struct {
+	keyID     string
 	masterKey [32]byte
 }
 
-// NewKMS returns a basic KMS implementation from a single 256 bit master key.
+// NewMasterKey returns a basic KMS implementation from a single 256 bit master key.
 //
 // The KMS accepts any keyID but binds the keyID and context cryptographically
 // to the generated keys.
-func NewKMS(key [32]byte) KMS { return &masterKeyKMS{masterKey: key} }
+func NewMasterKey(keyID string, key [32]byte) KMS { return &masterKeyKMS{keyID: keyID, masterKey: key} }
+
+func (kms *masterKeyKMS) KeyID() string {
+	return kms.keyID
+}
 
 func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
 	if _, err = io.ReadFull(rand.Reader, key[:]); err != nil {

cmd/crypto/kms_test.go

@@ -40,8 +40,9 @@ var masterKeyKMSTests = []struct {
 }
 
 func TestMasterKeyKMS(t *testing.T) {
-	kms := NewKMS([32]byte{})
 	for i, test := range masterKeyKMSTests {
+		kms := NewMasterKey(test.GenKeyID, [32]byte{})
+
 		key, sealedKey, err := kms.GenerateKey(test.GenKeyID, test.GenContext)
 		if err != nil {
 			t.Errorf("Test %d: KMS failed to generate key: %v", i, err)

cmd/crypto/metadata.go

@@ -17,6 +17,7 @@ package crypto
 import (
 	"context"
 	"encoding/base64"
+	"errors"
 	"fmt"
 
 	"github.com/minio/minio/cmd/logger"
@@ -119,28 +120,46 @@ func CreateMultipartMetadata(metadata map[string]string) map[string]string {
 	return metadata
 }
 
-// CreateMetadata encodes the keyID, the sealed kms data key and the sealed key
-// into the metadata and returns the modified metadata. It allocates a new
-// metadata map if metadata is nil.
+// CreateMetadata encodes the sealed object key into the metadata and returns
+// the modified metadata. If the keyID and the kmsKey is not empty it encodes
+// both into the metadata as well. It allocates a new metadata map if metadata
+// is nil.
 func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
 	if sealedKey.Algorithm != SealAlgorithm {
 		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
 	}
 
+	// There are two possibilites:
+	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the caller has passed an invalid argument combination.
+	if keyID == "" && len(kmsKey) != 0 {
+		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
+	}
+	if keyID != "" && len(kmsKey) == 0 {
+		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
+	}
+
 	if metadata == nil {
 		metadata = map[string]string{}
 	}
-	metadata[S3KMSKeyID] = keyID
+
 	metadata[SSESealAlgorithm] = sealedKey.Algorithm
 	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
 	metadata[S3SealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
-	metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
+		metadata[S3KMSKeyID] = keyID
+		metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	}
 	return metadata
 }
 
 // ParseMetadata extracts all SSE-S3 related values from the object metadata
-// and checks whether they are well-formed. It returns the KMS key-ID, the
-// sealed KMS key and the sealed object key on success.
+// and checks whether they are well-formed. It returns the sealed object key
+// on success. If the metadata contains both, a KMS master key ID and a sealed
+// KMS data key it returns both. If the metadata does not contain neither a
+// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
+// KMS data key. Otherwise, it returns an error.
 func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
 	// Extract all required values from object metadata
 	b64IV, ok := metadata[SSEIV]
@@ -153,15 +172,20 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 	b64SealedKey, ok := metadata[S3SealedKey]
 	if !ok {
-		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal sealed key for SSE-S3"}
+		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal sealed key for SSE-S3")
 	}
-	keyID, ok = metadata[S3KMSKeyID]
-	if !ok {
-		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal KMS key-ID for SSE-S3"}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be a key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the metadata is corrupted.
+	keyID, idPresent := metadata[S3KMSKeyID]
+	b64KMSSealedKey, kmsKeyPresent := metadata[S3KMSSealedKey]
+	if !idPresent && kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal KMS key-ID for SSE-S3")
 	}
-	b64KMSSealedKey, ok := metadata[S3KMSSealedKey]
-	if !ok {
-		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal sealed KMS data key for SSE-S3"}
+	if idPresent && !kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal sealed KMS data key for SSE-S3")
 	}
 
 	// Check whether all extracted values are well-formed
@@ -174,11 +198,13 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return keyID, kmsKey, sealedKey, Error{"The internal sealed key for SSE-S3 is invalid"}
+		return keyID, kmsKey, sealedKey, Error("The internal sealed key for SSE-S3 is invalid")
 	}
-	kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
-	if err != nil {
-		return keyID, kmsKey, sealedKey, Error{"The internal sealed KMS data key for SSE-S3 is invalid"}
+	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
+		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+		if err != nil {
+			return keyID, kmsKey, sealedKey, Error("The internal sealed KMS data key for SSE-S3 is invalid")
+		}
 	}
 
 	sealedKey.Algorithm = algorithm
@@ -218,7 +244,7 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	}
 	b64SealedKey, ok := metadata[SSECSealedKey]
 	if !ok {
-		return sealedKey, Error{"The object metadata is missing the internal sealed key for SSE-C"}
+		return sealedKey, Error("The object metadata is missing the internal sealed key for SSE-C")
 	}
 
 	// Check whether all extracted values are well-formed
@@ -231,7 +257,7 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return sealedKey, Error{"The internal sealed key for SSE-C is invalid"}
+		return sealedKey, Error("The internal sealed key for SSE-C is invalid")
 	}
 
 	sealedKey.Algorithm = algorithm

cmd/crypto/metadata_test.go

@@ -125,15 +125,15 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 1
 	{
-		ExpectedErr: Error{"The object metadata is missing the internal sealed key for SSE-S3"},
+		ExpectedErr: Error("The object metadata is missing the internal sealed key for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 2
 	{
-		ExpectedErr: Error{"The object metadata is missing the internal KMS key-ID for SSE-S3"},
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+		ExpectedErr: Error("The object metadata is missing the internal KMS key-ID for SSE-S3"),
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSSealedKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 3
 	{
-		ExpectedErr: Error{"The object metadata is missing the internal sealed KMS data key for SSE-S3"},
+		ExpectedErr: Error("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 4
@@ -150,7 +150,7 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 6
 	{
-		ExpectedErr: Error{"The internal sealed key for SSE-S3 is invalid"},
+		ExpectedErr: Error("The internal sealed key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
 			S3KMSKeyID: "", S3KMSSealedKey: "",
@@ -158,7 +158,7 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 7
 	{
-		ExpectedErr: Error{"The internal sealed KMS data key for SSE-S3 is invalid"},
+		ExpectedErr: Error("The internal sealed KMS data key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
 			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
@@ -218,7 +218,7 @@ var ssecParseMetadataTests = []struct {
 	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
 	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
 	{
-		ExpectedErr: Error{"The object metadata is missing the internal sealed key for SSE-C"},
+		ExpectedErr: Error("The object metadata is missing the internal sealed key for SSE-C"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
 	}, // 2
 	{
@@ -233,7 +233,7 @@ var ssecParseMetadataTests = []struct {
 		SealedKey: SealedKey{},
 	}, // 4
 	{
-		ExpectedErr: Error{"The internal sealed key for SSE-C is invalid"},
+		ExpectedErr: Error("The internal sealed key for SSE-C is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
 		},
@@ -287,7 +287,9 @@ var s3CreateMetadataTests = []struct {
 	SealedDataKey []byte
 	SealedKey     SealedKey
 }{
-	{KeyID: "", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+
+	{KeyID: "", SealedDataKey: nil, SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "my-minio-key", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
 	{KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
 	{KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}},
 }

cmd/crypto/parse.go

@@ -0,0 +1,42 @@
+// MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"encoding/hex"
+	"fmt"
+	"strings"
+)
+
+// ParseMasterKey parses the value of the environment variable
+// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
+func ParseMasterKey(envArg string) (KMS, error) {
+	values := strings.SplitN(envArg, ":", 2)
+	if len(values) != 2 {
+		return nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
+	}
+	var (
+		keyID  = values[0]
+		hexKey = values[1]
+	)
+	if len(hexKey) != 64 { // 2 hex bytes = 1 byte
+		return nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
+	}
+	var masterKey [32]byte
+	if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
+		return nil, err
+	}
+	return NewMasterKey(keyID, masterKey), nil
+}

cmd/crypto/parse_test.go

@@ -0,0 +1,59 @@
+// MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import "testing"
+
+func TestParseMasterKey(t *testing.T) {
+	tests := []struct {
+		envValue      string
+		expectedKeyID string
+		success       bool
+	}{
+		{
+			envValue: "invalid-value",
+			success:  false,
+		},
+		{
+			envValue: "too:many:colons",
+			success:  false,
+		},
+		{
+			envValue: "myminio-key:not-a-hex",
+			success:  false,
+		},
+		{
+			envValue:      "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574",
+			expectedKeyID: "my-minio-key",
+			success:       true,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.envValue, func(t *testing.T) {
+			kms, err := ParseMasterKey(tt.envValue)
+			if tt.success && err != nil {
+				t.Error(err)
+			}
+			if !tt.success && err == nil {
+				t.Error("Unexpected failure")
+			}
+			if err == nil && kms.KeyID() != tt.expectedKeyID {
+				t.Errorf("Expected keyID %s, got %s", tt.expectedKeyID, kms.KeyID())
+			}
+		})
+	}
+}

cmd/crypto/sse_test.go

@@ -191,7 +191,7 @@ var s3UnsealObjectKeyTests = []struct {
 	ExpectedErr error
 }{
 	{ // 0 - Valid KMS key-ID and valid metadata entries for bucket/object
-		KMS:    NewKMS([32]byte{}),
+		KMS:    NewMasterKey("my-minio-key", [32]byte{}),
 		Bucket: "bucket",
 		Object: "object",
 		Metadata: map[string]string{
@@ -204,7 +204,7 @@ var s3UnsealObjectKeyTests = []struct {
 		ExpectedErr: nil,
 	},
 	{ // 1 - Valid KMS key-ID for invalid metadata entries for bucket/object
-		KMS:    NewKMS([32]byte{}),
+		KMS:    NewMasterKey("my-minio-key", [32]byte{}),
 		Bucket: "bucket",
 		Object: "object",
 		Metadata: map[string]string{

cmd/crypto/vault.go

@@ -199,6 +199,11 @@ func (v *vaultService) authenticate() (err error) {
 	return
 }
 
+// KeyID - vault configured keyID
+func (v *vaultService) KeyID() string {
+	return v.config.Key.Name
+}
+
 // GenerateKey returns a new plaintext key, generated by the KMS,
 // and a sealed version of this plaintext key encrypted using the
 // named key referenced by keyID. It also binds the generated key

cmd/daily-lifecycle-ops.go

@@ -103,14 +103,12 @@ func startDailyLifecycle() {
 	}
 }
 
+var lifecycleTimeout = newDynamicTimeout(60*time.Second, time.Second)
+
 func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
-
-	zeroDuration := time.Millisecond
-	zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration)
-
 	// Lock to avoid concurrent lifecycle ops from other nodes
 	sweepLock := globalNSMutex.NewNSLock(ctx, "system", "daily-lifecycle-ops")
-	if err := sweepLock.GetLock(zeroDynamicTimeout); err != nil {
+	if err := sweepLock.GetLock(lifecycleTimeout); err != nil {
 		return err
 	}
 	defer sweepLock.Unlock()
@@ -141,22 +139,24 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 			if err != nil {
 				continue
 			}
+			var objects []string
 			for _, obj := range res.Objects {
 				// Find the action that need to be executed
 				action := l.ComputeAction(obj.Name, obj.ModTime)
 				switch action {
 				case lifecycle.DeleteAction:
-					objAPI.DeleteObject(ctx, bucket.Name, obj.Name)
+					objects = append(objects, obj.Name)
 				default:
-					// Nothing
-
+					// Do nothing, for now.
 				}
 			}
+			// Deletes a list of objects.
+			objAPI.DeleteObjects(ctx, bucket.Name, objects)
 			if !res.IsTruncated {
+				// We are done here, proceed to next bucket.
 				break
-			} else {
-				marker = res.NextMarker
 			}
+			marker = res.NextMarker
 		}
 	}
 

cmd/daily-sweeper.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"math/rand"
 	"sync"
 	"time"
 
@@ -48,15 +49,14 @@ func copyDailySweepListeners() []chan string {
 	return listenersCopy
 }
 
+var sweepTimeout = newDynamicTimeout(60*time.Second, time.Second)
+
 // sweepRound will list all objects, having read quorum or not and
 // feeds to all listeners, such as the background healing
 func sweepRound(ctx context.Context, objAPI ObjectLayer) error {
-	zeroDuration := time.Millisecond
-	zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration)
-
 	// General lock so we avoid parallel daily sweep by different instances.
 	sweepLock := globalNSMutex.NewNSLock(ctx, "system", "daily-sweep")
-	if err := sweepLock.GetLock(zeroDynamicTimeout); err != nil {
+	if err := sweepLock.GetLock(sweepTimeout); err != nil {
 		return err
 	}
 	defer sweepLock.Unlock()
@@ -76,10 +76,22 @@ func sweepRound(ctx context.Context, objAPI ObjectLayer) error {
 
 		marker := ""
 		for {
+			if globalHTTPServer != nil {
+				// Wait at max 10 minute for an inprogress request before proceeding to heal
+				waitCount := 600
+				// Any requests in progress, delay the heal.
+				for (globalHTTPServer.GetRequestCount() >= int32(globalXLSetCount*globalXLSetDriveCount)) &&
+					waitCount > 0 {
+					waitCount--
+					time.Sleep(1 * time.Second)
+				}
+			}
+
 			res, err := objAPI.ListObjectsHeal(ctx, bucket.Name, "", marker, "", 1000)
 			if err != nil {
 				continue
 			}
+
 			for _, obj := range res.Objects {
 				for _, l := range copyDailySweepListeners() {
 					l <- pathJoin(bucket.Name, obj.Name)
@@ -119,7 +131,9 @@ func dailySweeper() {
 		break
 	}
 
-	// Perform a sweep round each month
+	// Start with random sleep time, so as to avoid "synchronous checks" between servers
+	time.Sleep(time.Duration(rand.Float64() * float64(time.Hour)))
+
 	for {
 		if time.Since(lastSweepTime) < 30*24*time.Hour {
 			time.Sleep(time.Hour)
@@ -133,13 +147,12 @@ func dailySweeper() {
 			// instance doing the sweep round
 			case OperationTimedOut:
 				lastSweepTime = time.Now()
-			default:
-				logger.LogIf(ctx, err)
-				time.Sleep(time.Minute)
 				continue
 			}
-		} else {
-			lastSweepTime = time.Now()
+			logger.LogIf(ctx, err)
+			time.Sleep(time.Minute)
+			continue
 		}
+		lastSweepTime = time.Now()
 	}
 }

cmd/disk-cache-backend.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -33,8 +34,11 @@ import (
 	"time"
 
 	"github.com/djherbis/atime"
+	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/sio"
 	"github.com/ncw/directio"
 )
 
@@ -44,7 +48,9 @@ const (
 	cacheDataFile     = "part.1"
 	cacheMetaVersion  = "1.0.0"
 
-	cacheEnvDelimiter = ";"
+	// SSECacheEncrypted is the metadata key indicating that the object
+	// is a cache entry encrypted with cache KMS master key in globalCacheKMS.
+	SSECacheEncrypted = "X-Minio-Internal-Encrypted-Cache"
 )
 
 // CacheChecksumInfoV1 - carries checksums of individual blocks on disk.
@@ -81,7 +87,7 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 	o.ETag = extractETag(m.Meta)
 	o.ContentType = m.Meta["content-type"]
 	o.ContentEncoding = m.Meta["content-encoding"]
-	if storageClass, ok := m.Meta[amzStorageClass]; ok {
+	if storageClass, ok := m.Meta[xhttp.AmzStorageClass]; ok {
 		o.StorageClass = storageClass
 	} else {
 		o.StorageClass = globalMinioDefaultStorageClass
@@ -223,6 +229,7 @@ func (c *diskCache) purge() {
 					continue
 				}
 				cc := cacheControlOpts(objInfo)
+
 				if atime.Get(fi).Before(expiry) ||
 					cc.isStale(objInfo.ModTime) {
 					if err = removeAll(pathJoin(c.dir, obj.Name())); err != nil {
@@ -273,6 +280,10 @@ func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectI
 	}
 	oi.Bucket = bucket
 	oi.Name = object
+
+	if err = decryptCacheObjectETag(&oi); err != nil {
+		return oi, err
+	}
 	return
 }
 
@@ -323,7 +334,24 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 
 // Backend metadata could have changed through server side copy - reset cache metadata if that is the case
 func (c *diskCache) updateMetadataIfChanged(ctx context.Context, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo) error {
-	if !reflect.DeepEqual(bkObjectInfo.UserDefined, cacheObjInfo.UserDefined) ||
+
+	bkMeta := make(map[string]string)
+	cacheMeta := make(map[string]string)
+	for k, v := range bkObjectInfo.UserDefined {
+		if hasPrefix(k, ReservedMetadataPrefix) {
+			// Do not need to send any internal metadata
+			continue
+		}
+		bkMeta[http.CanonicalHeaderKey(k)] = v
+	}
+	for k, v := range cacheObjInfo.UserDefined {
+		if hasPrefix(k, ReservedMetadataPrefix) {
+			// Do not need to send any internal metadata
+			continue
+		}
+		cacheMeta[http.CanonicalHeaderKey(k)] = v
+	}
+	if !reflect.DeepEqual(bkMeta, cacheMeta) ||
 		bkObjectInfo.ETag != cacheObjInfo.ETag ||
 		bkObjectInfo.ContentType != cacheObjInfo.ContentType ||
 		bkObjectInfo.Expires != cacheObjInfo.Expires {
@@ -337,11 +365,11 @@ func getCacheSHADir(dir, bucket, object string) string {
 }
 
 // Cache data to disk with bitrot checksum added for each block of 1MB
-func (c *diskCache) bitrotWriteToCache(ctx context.Context, cachePath string, reader io.Reader, size int64) (int64, error) {
+func (c *diskCache) bitrotWriteToCache(ctx context.Context, cachePath string, reader io.Reader, size uint64) (int64, error) {
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
 		return 0, err
 	}
-	bufSize := int64(readSizeV1)
+	bufSize := uint64(readSizeV1)
 	if size > 0 && bufSize > size {
 		bufSize = size
 	}
@@ -367,28 +395,29 @@ func (c *diskCache) bitrotWriteToCache(ctx context.Context, cachePath string, re
 	bufp := c.pool.Get().(*[]byte)
 	defer c.pool.Put(bufp)
 
+	var n, n2 int
 	for {
-		n, err := io.ReadFull(reader, *bufp)
-		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF && err != io.ErrClosedPipe {
+		n, err = io.ReadFull(reader, *bufp)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			return 0, err
 		}
-		eof := err == io.EOF || err == io.ErrUnexpectedEOF || err == io.ErrClosedPipe
+		eof := err == io.EOF || err == io.ErrUnexpectedEOF
 		if n == 0 && size != 0 {
 			// Reached EOF, nothing more to be done.
 			break
 		}
 		h.Reset()
-		if _, err := h.Write((*bufp)[:n]); err != nil {
+		if _, err = h.Write((*bufp)[:n]); err != nil {
 			return 0, err
 		}
 		hashBytes := h.Sum(nil)
 		if _, err = f.Write(hashBytes); err != nil {
 			return 0, err
 		}
-		if _, err = f.Write((*bufp)[:n]); err != nil {
+		if n2, err = f.Write((*bufp)[:n]); err != nil {
 			return 0, err
 		}
-		bytesWritten += int64(n)
+		bytesWritten += int64(n2)
 		if eof {
 			break
 		}
@@ -396,6 +425,39 @@ func (c *diskCache) bitrotWriteToCache(ctx context.Context, cachePath string, re
 	return bytesWritten, nil
 }
 
+func newCacheEncryptReader(content io.Reader, bucket, object string, metadata map[string]string) (r io.Reader, err error) {
+	objectEncryptionKey, err := newCacheEncryptMetadata(bucket, object, metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey[:], MinVersion: sio.Version20})
+	if err != nil {
+		return nil, crypto.ErrInvalidCustomerKey
+	}
+	return reader, nil
+}
+func newCacheEncryptMetadata(bucket, object string, metadata map[string]string) ([]byte, error) {
+	var sealedKey crypto.SealedKey
+	if globalCacheKMS == nil {
+		return nil, errKMSNotConfigured
+	}
+	key, encKey, err := globalCacheKMS.GenerateKey(globalCacheKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return nil, err
+	}
+
+	objectKey := crypto.GenerateKey(key, rand.Reader)
+	sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
+	crypto.S3.CreateMetadata(metadata, globalCacheKMS.KeyID(), encKey, sealedKey)
+
+	if etag, ok := metadata["etag"]; ok {
+		metadata["etag"] = hex.EncodeToString(objectKey.SealETag([]byte(etag)))
+	}
+	metadata[SSECacheEncrypted] = ""
+	return objectKey[:], nil
+}
+
 // Caches the object to disk
 func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, opts ObjectOptions) error {
 	if c.diskUsageHigh() {
@@ -417,14 +479,33 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 		bufSize = size
 	}
 
-	n, err := c.bitrotWriteToCache(ctx, cachePath, data, size)
+	var metadata = make(map[string]string)
+	for k, v := range opts.UserDefined {
+		metadata[k] = v
+	}
+	var reader = data
+	var actualSize = uint64(size)
+	var err error
+	if globalCacheKMS != nil {
+		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
+		if err != nil {
+			return err
+		}
+		actualSize, _ = sio.EncryptedSize(uint64(size))
+	}
+	n, err := c.bitrotWriteToCache(ctx, cachePath, reader, actualSize)
 	if IsErr(err, baseErrs...) {
 		c.setOnline(false)
 	}
 	if err != nil {
+		removeAll(cachePath)
 		return err
 	}
-	return c.saveMetadata(ctx, bucket, object, opts.UserDefined, n)
+	if actualSize != uint64(n) {
+		removeAll(cachePath)
+		return IncompleteBody{}
+	}
+	return c.saveMetadata(ctx, bucket, object, metadata, n)
 }
 
 // checks streaming bitrot checksum of cached object before returning data
@@ -484,7 +565,6 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 		if _, err := io.ReadFull(rc, checksumHash); err != nil {
 			return err
 		}
-
 		h.Reset()
 		n, err := io.ReadFull(rc, *bufp)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
@@ -503,7 +583,8 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 		hashBytes := h.Sum(nil)
 
 		if !bytes.Equal(hashBytes, checksumHash) {
-			err = HashMismatchError{hex.EncodeToString(checksumHash), hex.EncodeToString(hashBytes)}
+			err = fmt.Errorf("hashes do not match expected %s, got %s",
+				hex.EncodeToString(checksumHash), hex.EncodeToString(hashBytes))
 			logger.LogIf(context.Background(), err)
 			return err
 		}
@@ -527,7 +608,7 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 	var objInfo ObjectInfo
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
 
-	if objInfo, err = c.statCache(ctx, cacheObjPath); err != nil {
+	if objInfo, err = c.Stat(ctx, bucket, object); err != nil {
 		return nil, toObjectErr(err, bucket, object)
 	}
 
@@ -547,12 +628,15 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 	filePath := path.Join(cacheObjPath, cacheDataFile)
 	pr, pw := io.Pipe()
 	go func() {
-		pw.CloseWithError(c.bitrotReadFromCache(ctx, filePath, off, length, pw))
+		err := c.bitrotReadFromCache(ctx, filePath, off, length, pw)
+		if err != nil {
+			removeAll(cacheObjPath)
+		}
+		pw.CloseWithError(err)
 	}()
 	// Cleanup function to cause the go routine above to exit, in
 	// case of incomplete read.
 	pipeCloser := func() { pr.Close() }
-
 	return fn(pr, h, opts.CheckCopyPrecondFn, pipeCloser)
 
 }

cmd/disk-cache-utils.go

@@ -17,8 +17,10 @@
 package cmd
 
 import (
+	"encoding/hex"
 	"io"
 	"os"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -124,7 +126,7 @@ func backendDownError(err error) bool {
 
 // IsCacheable returns if the object should be saved in the cache.
 func (o ObjectInfo) IsCacheable() bool {
-	return !crypto.IsEncrypted(o.UserDefined)
+	return !crypto.IsEncrypted(o.UserDefined) || globalCacheKMS != nil
 }
 
 // reads file cached on disk from offset upto length
@@ -147,6 +149,10 @@ func readCacheFileStream(filePath string, offset, length int64) (io.ReadCloser,
 		return nil, err
 	}
 
+	if err = os.Chtimes(filePath, time.Now(), st.ModTime()); err != nil {
+		return nil, err
+	}
+
 	// Verify if its not a regular file, since subsequent Seek is undefined.
 	if !st.Mode().IsRegular() {
 		return nil, errIsNotRegular
@@ -168,3 +174,47 @@ func readCacheFileStream(filePath string, offset, length int64) (io.ReadCloser,
 		io.Closer
 	}{Reader: io.LimitReader(fr, length), Closer: fr}, nil
 }
+
+func isCacheEncrypted(meta map[string]string) bool {
+	_, ok := meta[SSECacheEncrypted]
+	return ok
+}
+
+// decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
+func decryptCacheObjectETag(info *ObjectInfo) error {
+	// Directories are never encrypted.
+	if info.IsDir {
+		return nil
+	}
+	encrypted := crypto.S3.IsEncrypted(info.UserDefined) && isCacheEncrypted(info.UserDefined)
+
+	switch {
+	case encrypted:
+		if globalCacheKMS == nil {
+			return errKMSNotConfigured
+		}
+		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(info.UserDefined)
+
+		if err != nil {
+			return err
+		}
+		extKey, err := globalCacheKMS.UnsealKey(keyID, kmsKey, crypto.Context{info.Bucket: path.Join(info.Bucket, info.Name)})
+		if err != nil {
+			return err
+		}
+		var objectKey crypto.ObjectKey
+		if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), info.Bucket, info.Name); err != nil {
+			return err
+		}
+		etagStr := tryDecryptETag(objectKey[:], info.ETag, false)
+		// backend ETag was hex encoded before encrypting, so hex decode to get actual ETag
+		etag, err := hex.DecodeString(etagStr)
+		if err != nil {
+			return err
+		}
+		info.ETag = string(etag)
+		return nil
+	}
+
+	return nil
+}

cmd/disk-cache.go

@@ -13,7 +13,9 @@ import (
 	"time"
 
 	"github.com/djherbis/atime"
+	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
@@ -35,6 +37,7 @@ type CacheObjectLayer interface {
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
 	DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error)
+	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	// Storage operations.
 	StorageInfo(ctx context.Context) CacheStorageInfo
 }
@@ -58,6 +61,7 @@ type cacheObjects struct {
 	GetObjectInfoFn  func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn   func(ctx context.Context, bucket, object string) error
 	DeleteObjectsFn  func(ctx context.Context, bucket string, objects []string) ([]error, error)
+	PutObjectFn      func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 }
 
 func (c *cacheObjects) delete(ctx context.Context, dcache *diskCache, bucket, object string) (err error) {
@@ -231,13 +235,13 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	pipeReader, pipeWriter := io.Pipe()
 	teeReader := io.TeeReader(bkReader, pipeWriter)
 	go func() {
-		putErr := dcache.Put(ctx, bucket, object, io.LimitReader(pipeReader, bkReader.ObjInfo.Size), bkReader.ObjInfo.Size, ObjectOptions{UserDefined: getMetadata(bkReader.ObjInfo)})
+		putErr := c.put(ctx, dcache, bucket, object, io.LimitReader(pipeReader, bkReader.ObjInfo.Size), bkReader.ObjInfo.Size, ObjectOptions{UserDefined: getMetadata(bkReader.ObjInfo)})
 		// close the write end of the pipe, so the error gets
 		// propagated to getObjReader
 		pipeWriter.CloseWithError(putErr)
 	}()
 	cleanupBackend := func() { bkReader.Close() }
-	cleanupPipe := func() { pipeReader.Close() }
+	cleanupPipe := func() { pipeWriter.Close() }
 	return NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, opts.CheckCopyPrecondFn, cleanupBackend, cleanupPipe)
 }
 
@@ -263,7 +267,6 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 			return cachedObjInfo, nil
 		}
 	}
-
 	objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
 	if err != nil {
 		if _, ok := err.(ObjectNotFound); ok {
@@ -388,7 +391,7 @@ func (c *cacheObjects) hashIndex(bucket, object string) int {
 
 // newCache initializes the cacheFSObjects for the "drives" specified in config.json
 // or the global env overrides.
-func newCache(config CacheConfig) ([]*diskCache, bool, error) {
+func newCache(config cache.Config) ([]*diskCache, bool, error) {
 	var caches []*diskCache
 	ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
 	formats, migrating, err := loadAndValidateCacheFormat(ctx, config.Drives)
@@ -445,8 +448,9 @@ func checkAtimeSupport(dir string) (err error) {
 	return
 }
 func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
-	logger.StartupMessage(colorBlue("Cache migration initiated ...."))
-	var wg = &sync.WaitGroup{}
+	logStartupMessage(color.Blue("Cache migration initiated ...."))
+
+	var wg sync.WaitGroup
 	errs := make([]error, len(c.cache))
 	for i, dc := range c.cache {
 		if dc == nil {
@@ -480,11 +484,60 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 	c.migMutex.Lock()
 	defer c.migMutex.Unlock()
 	c.migrating = false
-	logger.StartupMessage(colorBlue("Cache migration completed successfully."))
+	logStartupMessage(color.Blue("Cache migration completed successfully."))
+}
+
+// PutObject - caches the uploaded object for single Put operations
+func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	putObjectFn := c.PutObjectFn
+	dcache, err := c.getCacheToLoc(ctx, bucket, object)
+	if err != nil {
+		// disk cache could not be located,execute backend call.
+		return putObjectFn(ctx, bucket, object, r, opts)
+	}
+	size := r.Size()
+	if c.skipCache() {
+		return putObjectFn(ctx, bucket, object, r, opts)
+	}
+
+	// fetch from backend if there is no space on cache drive
+	if !dcache.diskAvailable(size) {
+		return putObjectFn(ctx, bucket, object, r, opts)
+	}
+	if opts.ServerSideEncryption != nil {
+		return putObjectFn(ctx, bucket, object, r, opts)
+	}
+	// fetch from backend if cache exclude pattern or cache-control
+	// directive set to exclude
+	if c.isCacheExclude(bucket, object) {
+		dcache.Delete(ctx, bucket, object)
+		return putObjectFn(ctx, bucket, object, r, opts)
+	}
+
+	objInfo, err = putObjectFn(ctx, bucket, object, r, opts)
+
+	if err == nil {
+		go func() {
+			// fill cache in the background
+			bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
+			if bErr != nil {
+				return
+			}
+			defer bReader.Close()
+			oi, err := c.stat(ctx, dcache, bucket, object)
+			// avoid cache overwrite if another background routine filled cache
+			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
+				c.put(ctx, dcache, bucket, object, bReader, bReader.ObjInfo.Size, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
+			}
+		}()
+	}
+
+	return objInfo, err
+
 }
 
 // Returns cacheObjects for use by Server.
-func newServerCacheObjects(ctx context.Context, config CacheConfig) (CacheObjectLayer, error) {
+func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjectLayer, error) {
 	// list of disk caches for cache "drives" specified in config.json or MINIO_CACHE_DRIVES env var.
 	cache, migrateSw, err := newCache(config)
 	if err != nil {
@@ -513,6 +566,9 @@ func newServerCacheObjects(ctx context.Context, config CacheConfig) (CacheObject
 			}
 			return errs, nil
 		},
+		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
+		},
 	}
 	if migrateSw {
 		go c.migrateCacheFromV1toV2(ctx)

cmd/encryption-v1.go

@@ -32,7 +32,6 @@ import (
 	"github.com/minio/minio-go/v6/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/ioutil"
 	sha256 "github.com/minio/sha256-simd"
 	"github.com/minio/sio"
 )
@@ -64,23 +63,6 @@ const (
 
 )
 
-const (
-	// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
-	// hash function. The key derivation of DARE-SHA256 is not optimal and does not include the object path.
-	// It is considered legacy and should not be used anymore.
-	SSESealAlgorithmDareSha256 = "DARE-SHA256"
-
-	// SSESealAlgorithmDareV2HmacSha256 specifies DAREv2 as authenticated en/decryption scheme and SHA256 as cryptographic
-	// hash function for the HMAC PRF.
-	SSESealAlgorithmDareV2HmacSha256 = "DAREv2-HMAC-SHA256"
-)
-
-// hasServerSideEncryptionHeader returns true if the given HTTP header
-// contains server-side-encryption.
-func hasServerSideEncryptionHeader(header http.Header) bool {
-	return crypto.S3.IsRequested(header) || crypto.SSEC.IsRequested(header)
-}
-
 // isEncryptedMultipart returns true if the current object is
 // uploaded by the user using multipart mechanism:
 // initiate new multipart, upload part, complete upload
@@ -175,12 +157,12 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 			return err
 		}
 
-		newKey, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
+		newKey, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return err
 		}
 		sealedKey = objectKey.Seal(newKey, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
-		crypto.S3.CreateMetadata(metadata, globalKMSKeyID, encKey, sealedKey)
+		crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
 		return nil
 	}
 }
@@ -191,14 +173,14 @@ func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]s
 		if GlobalKMS == nil {
 			return nil, errKMSNotConfigured
 		}
-		key, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
+		key, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return nil, err
 		}
 
 		objectKey := crypto.GenerateKey(key, rand.Reader)
 		sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
-		crypto.S3.CreateMetadata(metadata, globalKMSKeyID, encKey, sealedKey)
+		crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
 		return objectKey[:], nil
 	}
 	var extKey [32]byte
@@ -279,6 +261,23 @@ func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]st
 	switch {
 	default:
 		return nil, errObjectTampered
+	case crypto.S3.IsEncrypted(metadata) && isCacheEncrypted(metadata):
+		if globalCacheKMS == nil {
+			return nil, errKMSNotConfigured
+		}
+		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
+		if err != nil {
+			return nil, err
+		}
+		extKey, err := globalCacheKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
+		if err != nil {
+			return nil, err
+		}
+		var objectKey crypto.ObjectKey
+		if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
+			return nil, err
+		}
+		return objectKey[:], nil
 	case crypto.S3.IsEncrypted(metadata):
 		if GlobalKMS == nil {
 			return nil, errKMSNotConfigured
@@ -580,236 +579,6 @@ func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
 	return len(p), nil
 }
 
-// DecryptBlocksWriter - decrypts  multipart parts, while implementing
-// a io.Writer compatible interface.
-type DecryptBlocksWriter struct {
-	// Original writer where the plain data will be written
-	writer io.Writer
-	// Current decrypter for the current encrypted data block
-	decrypter io.WriteCloser
-	// Start sequence number
-	startSeqNum uint32
-	// Current part index
-	partIndex int
-	// Parts information
-	parts          []ObjectPartInfo
-	req            *http.Request
-	bucket, object string
-	metadata       map[string]string
-
-	partEncRelOffset int64
-
-	copySource bool
-	// Customer Key
-	customerKeyHeader string
-}
-
-func (w *DecryptBlocksWriter) buildDecrypter(partID int) error {
-	m := make(map[string]string)
-	for k, v := range w.metadata {
-		m[k] = v
-	}
-	// Initialize the first decrypter, new decrypters will be initialized in Write() operation as needed.
-	var key []byte
-	var err error
-	if w.copySource {
-		if crypto.SSEC.IsEncrypted(w.metadata) {
-			w.req.Header.Set(crypto.SSECopyKey, w.customerKeyHeader)
-			key, err = ParseSSECopyCustomerRequest(w.req.Header, w.metadata)
-		}
-	} else {
-		if crypto.SSEC.IsEncrypted(w.metadata) {
-			w.req.Header.Set(crypto.SSECKey, w.customerKeyHeader)
-			key, err = ParseSSECustomerRequest(w.req)
-		}
-	}
-	if err != nil {
-		return err
-	}
-
-	objectEncryptionKey, err := decryptObjectInfo(key, w.bucket, w.object, m)
-	if err != nil {
-		return err
-	}
-
-	var partIDbin [4]byte
-	binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID
-
-	mac := hmac.New(sha256.New, objectEncryptionKey) // derive part encryption key from part ID and object key
-	mac.Write(partIDbin[:])
-	partEncryptionKey := mac.Sum(nil)
-
-	// make sure to provide a NopCloser such that a Close
-	// on sio.decryptWriter doesn't close the underlying writer's
-	// close which perhaps can close the stream prematurely.
-	decrypter, err := newDecryptWriterWithObjectKey(ioutil.NopCloser(w.writer), partEncryptionKey, w.startSeqNum, m)
-	if err != nil {
-		return err
-	}
-
-	if w.decrypter != nil {
-		// Pro-actively close the writer such that any pending buffers
-		// are flushed already before we allocate a new decrypter.
-		err = w.decrypter.Close()
-		if err != nil {
-			return err
-		}
-	}
-
-	w.decrypter = decrypter
-	return nil
-}
-
-func (w *DecryptBlocksWriter) Write(p []byte) (int, error) {
-	var err error
-	var n1 int
-	if int64(len(p)) < w.parts[w.partIndex].Size-w.partEncRelOffset {
-		n1, err = w.decrypter.Write(p)
-		if err != nil {
-			return 0, err
-		}
-		w.partEncRelOffset += int64(n1)
-	} else {
-		n1, err = w.decrypter.Write(p[:w.parts[w.partIndex].Size-w.partEncRelOffset])
-		if err != nil {
-			return 0, err
-		}
-
-		// We should now proceed to next part, reset all values appropriately.
-		w.partEncRelOffset = 0
-		w.startSeqNum = 0
-
-		w.partIndex++
-
-		err = w.buildDecrypter(w.partIndex + 1)
-		if err != nil {
-			return 0, err
-		}
-
-		n1, err = w.decrypter.Write(p[n1:])
-		if err != nil {
-			return 0, err
-		}
-
-		w.partEncRelOffset += int64(n1)
-	}
-
-	return len(p), nil
-}
-
-// Close closes the LimitWriter. It behaves like io.Closer.
-func (w *DecryptBlocksWriter) Close() error {
-	if w.decrypter != nil {
-		err := w.decrypter.Close()
-		if err != nil {
-			return err
-		}
-	}
-
-	if closer, ok := w.writer.(io.Closer); ok {
-		return closer.Close()
-	}
-	return nil
-}
-
-// DecryptAllBlocksCopyRequest - setup a struct which can decrypt many concatenated encrypted data
-// parts information helps to know the boundaries of each encrypted data block, this function decrypts
-// all parts starting from part-1.
-func DecryptAllBlocksCopyRequest(client io.Writer, r *http.Request, bucket, object string, objInfo ObjectInfo) (io.WriteCloser, int64, error) {
-	w, _, size, err := DecryptBlocksRequest(client, r, bucket, object, 0, objInfo.Size, objInfo, true)
-	return w, size, err
-}
-
-// DecryptBlocksRequest - setup a struct which can decrypt many concatenated encrypted data
-// parts information helps to know the boundaries of each encrypted data block.
-func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object string, startOffset, length int64, objInfo ObjectInfo, copySource bool) (io.WriteCloser, int64, int64, error) {
-	var seqNumber uint32
-	var encStartOffset, encLength int64
-
-	if !isEncryptedMultipart(objInfo) {
-		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
-
-		var writer io.WriteCloser
-		var err error
-		if copySource {
-			writer, err = DecryptCopyRequest(client, r, bucket, object, objInfo.UserDefined)
-		} else {
-			writer, err = DecryptRequestWithSequenceNumber(client, r, bucket, object, seqNumber, objInfo.UserDefined)
-		}
-		if err != nil {
-			return nil, 0, 0, err
-		}
-		return writer, encStartOffset, encLength, nil
-	}
-
-	_, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
-
-	var partStartIndex int
-	var partStartOffset = startOffset
-	// Skip parts until final offset maps to a particular part offset.
-	for i, part := range objInfo.Parts {
-		decryptedSize, err := sio.DecryptedSize(uint64(part.Size))
-		if err != nil {
-			return nil, -1, -1, errObjectTampered
-		}
-
-		partStartIndex = i
-
-		// Offset is smaller than size we have reached the
-		// proper part offset, break out we start from
-		// this part index.
-		if partStartOffset < int64(decryptedSize) {
-			break
-		}
-
-		// Continue to look for next part.
-		partStartOffset -= int64(decryptedSize)
-	}
-
-	startSeqNum := partStartOffset / SSEDAREPackageBlockSize
-	partEncRelOffset := int64(startSeqNum) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
-
-	w := &DecryptBlocksWriter{
-		writer:            client,
-		startSeqNum:       uint32(startSeqNum),
-		partEncRelOffset:  partEncRelOffset,
-		parts:             objInfo.Parts,
-		partIndex:         partStartIndex,
-		req:               r,
-		bucket:            bucket,
-		object:            object,
-		customerKeyHeader: r.Header.Get(crypto.SSECKey),
-		copySource:        copySource,
-	}
-
-	w.metadata = map[string]string{}
-	// Copy encryption metadata for internal use.
-	for k, v := range objInfo.UserDefined {
-		w.metadata[k] = v
-	}
-
-	// Purge all the encryption headers.
-	delete(objInfo.UserDefined, crypto.SSEIV)
-	delete(objInfo.UserDefined, crypto.SSESealAlgorithm)
-	delete(objInfo.UserDefined, crypto.SSECSealedKey)
-	delete(objInfo.UserDefined, crypto.SSEMultipart)
-
-	if crypto.S3.IsEncrypted(objInfo.UserDefined) {
-		delete(objInfo.UserDefined, crypto.S3SealedKey)
-		delete(objInfo.UserDefined, crypto.S3KMSKeyID)
-		delete(objInfo.UserDefined, crypto.S3KMSSealedKey)
-	}
-	if w.copySource {
-		w.customerKeyHeader = r.Header.Get(crypto.SSECopyKey)
-	}
-
-	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
-		return nil, 0, 0, err
-	}
-
-	return w, encStartOffset, encLength, nil
-}
-
 // getEncryptedMultipartsOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
 func getEncryptedMultipartsOffsetLength(offset, length int64, obj ObjectInfo) (uint32, int64, int64) {
 	// Calculate encrypted offset of a multipart object

cmd/encryption-v1_test.go

@@ -28,86 +28,6 @@ import (
 	"github.com/minio/sio"
 )
 
-var hasServerSideEncryptionHeaderTests = []struct {
-	headers    map[string]string
-	sseRequest bool
-}{
-	{headers: map[string]string{crypto.SSECAlgorithm: "AES256", crypto.SSECKey: "key", crypto.SSECKeyMD5: "md5"}, sseRequest: true},                             // 0
-	{headers: map[string]string{crypto.SSECAlgorithm: "AES256"}, sseRequest: true},                                                                              // 1
-	{headers: map[string]string{crypto.SSECKey: "key"}, sseRequest: true},                                                                                       // 2
-	{headers: map[string]string{crypto.SSECKeyMD5: "md5"}, sseRequest: true},                                                                                    // 3
-	{headers: map[string]string{}, sseRequest: false},                                                                                                           // 4
-	{headers: map[string]string{crypto.SSECopyAlgorithm + " ": "AES256", " " + crypto.SSECopyKey: "key", crypto.SSECopyKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
-	{headers: map[string]string{crypto.SSECopyAlgorithm: "", crypto.SSECopyKey: "", crypto.SSECopyKeyMD5: ""}, sseRequest: false},                               // 6
-	{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: true},                                                                                        // 7
-}
-
-func TestHasServerSideEncryptionHeader(t *testing.T) {
-	for i, test := range hasServerSideEncryptionHeaderTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		if hasServerSideEncryptionHeader(headers) != test.sseRequest {
-			t.Errorf("Test %d: Expected hasServerSideEncryptionHeader to return %v", i, test.sseRequest)
-		}
-	}
-}
-
-var hasSSECopyCustomerHeaderTests = []struct {
-	headers    map[string]string
-	sseRequest bool
-}{
-	{headers: map[string]string{crypto.SSECopyAlgorithm: "AES256", crypto.SSECopyKey: "key", crypto.SSECopyKeyMD5: "md5"}, sseRequest: true},                    // 0
-	{headers: map[string]string{crypto.SSECopyAlgorithm: "AES256"}, sseRequest: true},                                                                           // 1
-	{headers: map[string]string{crypto.SSECopyKey: "key"}, sseRequest: true},                                                                                    // 2
-	{headers: map[string]string{crypto.SSECopyKeyMD5: "md5"}, sseRequest: true},                                                                                 // 3
-	{headers: map[string]string{}, sseRequest: false},                                                                                                           // 4
-	{headers: map[string]string{crypto.SSECopyAlgorithm + " ": "AES256", " " + crypto.SSECopyKey: "key", crypto.SSECopyKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
-	{headers: map[string]string{crypto.SSECopyAlgorithm: "", crypto.SSECopyKey: "", crypto.SSECopyKeyMD5: ""}, sseRequest: true},                                // 6
-	{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: false},                                                                                       // 7
-
-}
-
-func TestIsSSECopyCustomerRequest(t *testing.T) {
-	for i, test := range hasSSECopyCustomerHeaderTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		if crypto.SSECopy.IsRequested(headers) != test.sseRequest {
-			t.Errorf("Test %d: Expected crypto.SSECopy.IsRequested to return %v", i, test.sseRequest)
-		}
-	}
-}
-
-var hasSSECustomerHeaderTests = []struct {
-	headers    map[string]string
-	sseRequest bool
-}{
-	{headers: map[string]string{crypto.SSECAlgorithm: "AES256", crypto.SSECKey: "key", crypto.SSECKeyMD5: "md5"}, sseRequest: true},                    // 0
-	{headers: map[string]string{crypto.SSECAlgorithm: "AES256"}, sseRequest: true},                                                                     // 1
-	{headers: map[string]string{crypto.SSECKey: "key"}, sseRequest: true},                                                                              // 2
-	{headers: map[string]string{crypto.SSECKeyMD5: "md5"}, sseRequest: true},                                                                           // 3
-	{headers: map[string]string{}, sseRequest: false},                                                                                                  // 4
-	{headers: map[string]string{crypto.SSECAlgorithm + " ": "AES256", " " + crypto.SSECKey: "key", crypto.SSECKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
-	{headers: map[string]string{crypto.SSECAlgorithm: "", crypto.SSECKey: "", crypto.SSECKeyMD5: ""}, sseRequest: true},                                // 6
-	{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: false},                                                                              // 7
-
-}
-
-func TestHasSSECustomerHeader(t *testing.T) {
-	for i, test := range hasSSECustomerHeaderTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		if crypto.SSEC.IsRequested(headers) != test.sseRequest {
-			t.Errorf("Test %d: Expected hasSSECustomerHeader to return %v", i, test.sseRequest)
-		}
-	}
-}
-
 var encryptRequestTests = []struct {
 	header   map[string]string
 	metadata map[string]string
@@ -173,7 +93,7 @@ var decryptRequestTests = []struct {
 			crypto.SSECKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
 		},
 		metadata: map[string]string{
-			crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
+			crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
 			crypto.SSEIV:            "7nQqotA8xgrPx6QK7Ap3GCfjKitqJSrGP7xzgErSJlw=",
 			crypto.SSECSealedKey:    "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA==",
 		},
@@ -188,7 +108,7 @@ var decryptRequestTests = []struct {
 			crypto.SSECKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
 		},
 		metadata: map[string]string{
-			crypto.SSESealAlgorithm: SSESealAlgorithmDareV2HmacSha256,
+			crypto.SSESealAlgorithm: crypto.SealAlgorithm,
 			crypto.SSEIV:            "qEqmsONcorqlcZXJxaw32H04eyXyXwUgjHzlhkaIYrU=",
 			crypto.SSECSealedKey:    "IAAfAIM14ugTGcM/dIrn4iQMrkl1sjKyeBQ8FBEvRebYj8vWvxG+0cJRpC6NXRU1wJN50JaUOATjO7kz0wZ2mA==",
 		},
@@ -218,7 +138,7 @@ var decryptRequestTests = []struct {
 			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
 		},
 		metadata: map[string]string{
-			crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
+			crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
 			crypto.SSEIV:            "RrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
 			crypto.SSECSealedKey:    "SY5E9AvI2tI7/nUrUAssIGE32Hcs4rR9z/CUuPqu5N4=",
 		},
@@ -233,7 +153,7 @@ var decryptRequestTests = []struct {
 			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
 		},
 		metadata: map[string]string{
-			crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
+			crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
 			crypto.SSEIV:            "XAm0dRrJsEsyPb1UuFNezv1bl9ehxuYsgUVC/MUctE2k=",
 			crypto.SSECSealedKey:    "SY5E9AvI2tI7/nUrUAssIGE32Hds4rR9z/CUuPqu5N4=",
 		},
@@ -248,7 +168,7 @@ var decryptRequestTests = []struct {
 			crypto.SSECKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
 		},
 		metadata: map[string]string{
-			crypto.SSESealAlgorithm: SSESealAlgorithmDareV2HmacSha256,
+			crypto.SSESealAlgorithm: crypto.SealAlgorithm,
 			crypto.SSEIV:            "qEqmsONcorqlcZXJxaw32H04eyXyXwUgjHzlhkaIYrU=",
 			crypto.SSECSealedKey:    "IAAfAIM14ugTGcM/dIrn4iQMrkl1sjKyeBQ8FBEvRebYj8vWvxG+0cJRpC6NXRU1wJN50JaUOATjO7kz0wZ2mA==",
 		},
@@ -298,12 +218,12 @@ var decryptObjectInfoTests = []struct {
 		expErr:  nil,
 	},
 	{
-		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
 		expErr:  nil,
 	},
 	{
-		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
+		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
 		expErr:  nil,
 	},
@@ -318,7 +238,7 @@ var decryptObjectInfoTests = []struct {
 		expErr:  errInvalidEncryptionParameters,
 	},
 	{
-		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
+		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
 		expErr:  errObjectTampered,
 	},
@@ -408,7 +328,7 @@ func TestGetDecryptedRange(t *testing.T) {
 		}
 		udMap = func(isMulti bool) map[string]string {
 			m := map[string]string{
-				crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
+				crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
 				crypto.SSEMultipart:     "1",
 			}
 			if !isMulti {

cmd/endpoint-ellipses.go

@@ -18,12 +18,13 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 	"strconv"
 	"strings"
 
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/ellipses"
+	"github.com/minio/minio/pkg/env"
 )
 
 // This file implements and supports ellipses pattern for
@@ -77,13 +78,13 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 	}
 
 	var customSetDriveCount uint64
-	if v := os.Getenv("MINIO_ERASURE_SET_DRIVE_COUNT"); v != "" {
+	if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
 		customSetDriveCount, err = strconv.ParseUint(v, 10, 64)
 		if err != nil {
-			return nil, uiErrInvalidErasureSetSize(err)
+			return nil, config.ErrInvalidErasureSetSize(err)
 		}
 		if !isValidSetSize(customSetDriveCount) {
-			return nil, uiErrInvalidErasureSetSize(nil)
+			return nil, config.ErrInvalidErasureSetSize(nil)
 		}
 	}
 
@@ -91,7 +92,7 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 	for _, totalSize := range totalSizes {
 		// Check if totalSize has minimum range upto setSize
 		if totalSize < setSizes[0] || totalSize < customSetDriveCount {
-			return nil, uiErrInvalidNumberOfErasureEndpoints(nil)
+			return nil, config.ErrInvalidNumberOfErasureEndpoints(nil)
 		}
 	}
 
@@ -125,17 +126,17 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 	if customSetDriveCount > 0 {
 		msg := fmt.Sprintf("Invalid set drive count, leads to non-uniform distribution for the given number of disks. Possible values for custom set count are %d", possibleSetCounts(setSize))
 		if customSetDriveCount > setSize {
-			return nil, uiErrInvalidErasureSetSize(nil).Msg(msg)
+			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
 		if setSize%customSetDriveCount != 0 {
-			return nil, uiErrInvalidErasureSetSize(nil).Msg(msg)
+			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
 		setSize = customSetDriveCount
 	}
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		return nil, uiErrInvalidNumberOfErasureEndpoints(nil)
+		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil)
 	}
 
 	for i := range totalSizes {
@@ -198,14 +199,14 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) {
 	for i, arg := range args {
 		patterns, perr := ellipses.FindEllipsesPatterns(arg)
 		if perr != nil {
-			return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(perr.Error())
+			return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(perr.Error())
 		}
 		argPatterns[i] = patterns
 	}
 
 	ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns))
 	if err != nil {
-		return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+		return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}
 
 	ep.argPatterns = argPatterns
@@ -254,7 +255,7 @@ func GetAllSets(args ...string) ([][]string, error) {
 	for _, sargs := range setArgs {
 		for _, arg := range sargs {
 			if uniqueArgs.Contains(arg) {
-				return nil, uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args))
+				return nil, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args))
 			}
 			uniqueArgs.Add(arg)
 		}

cmd/endpoint.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,9 +20,7 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"net/http"
 	"net/url"
-	"os"
 	"path"
 	"path/filepath"
 	"runtime"
@@ -33,10 +31,10 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/cpu"
-	"github.com/minio/minio/pkg/disk"
-	"github.com/minio/minio/pkg/mem"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/mountinfo"
 )
 
@@ -229,7 +227,7 @@ func (endpoints EndpointList) UpdateIsLocal() error {
 	keepAliveTicker := time.NewTicker(retryInterval * time.Second)
 	defer keepAliveTicker.Stop()
 	for {
-		// Break if the local endpoint is found already. Or all the endpoints are resolved.
+		// Break if the local endpoint is found already Or all the endpoints are resolved.
 		if foundLocal || (epsResolved == len(endpoints)) {
 			break
 		}
@@ -242,13 +240,13 @@ func (endpoints EndpointList) UpdateIsLocal() error {
 		default:
 			for i, resolved := range resolvedList {
 				if resolved {
+					// Continue if host is already resolved.
 					continue
 				}
 
 				// return err if not Docker or Kubernetes
-				// We use IsDocker() method to check for Docker Swarm environment
-				// as there is no reliable way to clearly identify Swarm from
-				// Docker environment.
+				// We use IsDocker() to check for Docker environment
+				// We use IsKubernetes() to check for Kubernetes environment
 				isLocal, err := isLocalHost(endpoints[i].HostName)
 				if err != nil {
 					if !IsDocker() && !IsKubernetes() {
@@ -258,8 +256,7 @@ func (endpoints EndpointList) UpdateIsLocal() error {
 					timeElapsed := time.Since(startTime)
 					// log error only if more than 1s elapsed
 					if timeElapsed > time.Second {
-						// log the message to console about the host not being
-						// resolveable.
+						// Log the message to console about the host not being resolveable.
 						reqInfo := (&logger.ReqInfo{}).AppendTags("host", endpoints[i].HostName)
 						reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", ""))
 						ctx := logger.SetReqInfo(context.Background(), reqInfo)
@@ -276,98 +273,33 @@ func (endpoints EndpointList) UpdateIsLocal() error {
 			}
 
 			// Wait for the tick, if the there exist a local endpoint in discovery.
-			// Non docker/kubernetes environment does not need to wait.
-			if !foundLocal && (IsDocker() && IsKubernetes()) {
+			// Non docker/kubernetes environment we do not need to wait.
+			if !foundLocal && (IsDocker() || IsKubernetes()) {
 				<-keepAliveTicker.C
 			}
 		}
 	}
 
+	// On Kubernetes/Docker setups DNS resolves inappropriately sometimes
+	// where there are situations same endpoints with multiple disks
+	// come online indicating either one of them is local and some
+	// of them are not local. This situation can never happen and
+	// its only a possibility in orchestrated deployments with dynamic
+	// DNS. Following code ensures that we treat if one of the endpoint
+	// says its local for a given host - it is true for all endpoints
+	// for the same host. Following code ensures that this assumption
+	// is true and it works in all scenarios and it is safe to assume
+	// for a given host.
+	endpointLocalMap := make(map[string]bool)
+	for _, ep := range endpoints {
+		if ep.IsLocal {
+			endpointLocalMap[ep.Host] = ep.IsLocal
+		}
+	}
+	for i := range endpoints {
+		endpoints[i].IsLocal = endpointLocalMap[endpoints[i].Host]
+	}
 	return nil
-
-}
-
-// localEndpointsMemUsage - returns ServerMemUsageInfo for only the
-// local endpoints from given list of endpoints
-func localEndpointsMemUsage(endpoints EndpointList, r *http.Request) ServerMemUsageInfo {
-	var memUsages []mem.Usage
-	var historicUsages []mem.Usage
-	scratchSpace := map[string]bool{}
-	for _, endpoint := range endpoints {
-		// Only proceed for local endpoints
-		if endpoint.IsLocal {
-			if _, ok := scratchSpace[endpoint.Host]; ok {
-				continue
-			}
-			memUsages = append(memUsages, mem.GetUsage())
-			historicUsages = append(historicUsages, mem.GetHistoricUsage())
-			scratchSpace[endpoint.Host] = true
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpoints)
-	}
-	return ServerMemUsageInfo{
-		Addr:          addr,
-		Usage:         memUsages,
-		HistoricUsage: historicUsages,
-	}
-}
-
-// localEndpointsCPULoad - returns ServerCPULoadInfo for only the
-// local endpoints from given list of endpoints
-func localEndpointsCPULoad(endpoints EndpointList, r *http.Request) ServerCPULoadInfo {
-	var cpuLoads []cpu.Load
-	var historicLoads []cpu.Load
-	scratchSpace := map[string]bool{}
-	for _, endpoint := range endpoints {
-		// Only proceed for local endpoints
-		if endpoint.IsLocal {
-			if _, ok := scratchSpace[endpoint.Host]; ok {
-				continue
-			}
-			cpuLoads = append(cpuLoads, cpu.GetLoad())
-			historicLoads = append(historicLoads, cpu.GetHistoricLoad())
-			scratchSpace[endpoint.Host] = true
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpoints)
-	}
-	return ServerCPULoadInfo{
-		Addr:         addr,
-		Load:         cpuLoads,
-		HistoricLoad: historicLoads,
-	}
-}
-
-// localEndpointsDrivePerf - returns ServerDrivesPerfInfo for only the
-// local endpoints from given list of endpoints
-func localEndpointsDrivePerf(endpoints EndpointList, r *http.Request) ServerDrivesPerfInfo {
-	var dps []disk.Performance
-	for _, endpoint := range endpoints {
-		// Only proceed for local endpoints
-		if endpoint.IsLocal {
-			if _, err := os.Stat(endpoint.Path); err != nil {
-				// Since this drive is not available, add relevant details and proceed
-				dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
-				continue
-			}
-			dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()))
-			dp.Path = endpoint.Path
-			dps = append(dps, dp)
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpoints)
-	}
-	return ServerDrivesPerfInfo{
-		Addr: addr,
-		Perf: dps,
-	}
 }
 
 // NewEndpointList - returns new endpoint list based on input args.
@@ -464,14 +396,14 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			return serverAddr, endpoints, setupType, err
 		}
 		if endpoint.Type() != PathEndpointType {
-			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
+			return serverAddr, endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
 		}
 		endpoints = append(endpoints, endpoint)
 		setupType = FSSetupType
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(endpoints); err != nil {
-			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg(err.Error())
+			return serverAddr, endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg(err.Error())
 		}
 		return serverAddr, endpoints, setupType, nil
 	}
@@ -482,12 +414,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		var eps EndpointList
 		eps, err = NewEndpointList(iargs...)
 		if err != nil {
-			return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+			return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(eps); err != nil {
-			return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+			return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
 		for _, ep := range eps {
@@ -504,7 +436,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 	}
 
 	if err := endpoints.UpdateIsLocal(); err != nil {
-		return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+		return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}
 
 	// Here all endpoints are URL style.
@@ -532,7 +464,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 
 	// No local endpoint found.
 	if localEndpointCount == 0 {
-		return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg("no endpoint pointing to the local machine is found")
+		return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("no endpoint pointing to the local machine is found")
 	}
 
 	// Check whether same path is not used in endpoints of a host on different port.
@@ -548,7 +480,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
 				if !IPSet.Intersection(hostIPSet).IsEmpty() {
 					return serverAddr, endpoints, setupType,
-						uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
+						config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
 				}
 				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
 			} else {
@@ -566,7 +498,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			}
 			if localPathSet.Contains(endpoint.Path) {
 				return serverAddr, endpoints, setupType,
-					uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
+					config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
 			}
 			localPathSet.Add(endpoint.Path)
 		}
@@ -577,10 +509,10 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		if !localPortSet.Contains(serverAddrPort) {
 			if len(localPortSet) > 1 {
 				return serverAddr, endpoints, setupType,
-					uiErrInvalidErasureEndpoints(nil).Msg("port number in server address must match with one of the port in local endpoints")
+					config.ErrInvalidErasureEndpoints(nil).Msg("port number in server address must match with one of the port in local endpoints")
 			}
 			return serverAddr, endpoints, setupType,
-				uiErrInvalidErasureEndpoints(nil).Msg("server address and local endpoint have different ports")
+				config.ErrInvalidErasureEndpoints(nil).Msg("server address and local endpoint have different ports")
 		}
 	}
 
@@ -658,9 +590,9 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		return serverAddr, endpoints, setupType, err
 	}
 
-	_, dok := os.LookupEnv("MINIO_DOMAIN")
-	_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
-	_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	_, dok := env.Lookup(config.EnvDomain)
+	_, eok := env.Lookup(etcd.EnvEtcdEndpoints)
+	_, iok := env.Lookup(config.EnvPublicIPs)
 	if dok && eok && !iok {
 		updateDomainIPs(uniqueArgs)
 	}

cmd/endpoint_test.go

@@ -350,27 +350,29 @@ func TestCreateEndpoints(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
-
-		if err == nil {
-			if testCase.expectedErr != nil {
-				t.Fatalf("Test (%d) error: expected = %v, got = <nil>", i+1, testCase.expectedErr)
-			} else {
-				if serverAddr != testCase.expectedServerAddr {
-					t.Fatalf("Test (%d) serverAddr: expected = %v, got = %v", i+1, testCase.expectedServerAddr, serverAddr)
-				}
-				if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
-					t.Fatalf("Test (%d) endpoints: expected = %v, got = %v", i+1, testCase.expectedEndpoints, endpoints)
-				}
-				if setupType != testCase.expectedSetupType {
-					t.Fatalf("Test (%d) setupType: expected = %v, got = %v", i+1, testCase.expectedSetupType, setupType)
+		testCase := testCase
+		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
+			serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
+			if err == nil {
+				if testCase.expectedErr != nil {
+					t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+				} else {
+					if serverAddr != testCase.expectedServerAddr {
+						t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
+					}
+					if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
+						t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
+					}
+					if setupType != testCase.expectedSetupType {
+						t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
+					}
 				}
+			} else if testCase.expectedErr == nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			} else if err.Error() != testCase.expectedErr.Error() {
+				t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
 			}
-		} else if testCase.expectedErr == nil {
-			t.Fatalf("Test (%d) error: expected = <nil>, got = %v", i+1, err)
-		} else if err.Error() != testCase.expectedErr.Error() {
-			t.Fatalf("Test (%d) error: expected = %v, got = %v", i+1, testCase.expectedErr, err)
-		}
+		})
 	}
 }
 

cmd/environment.go

@@ -1,182 +0,0 @@
-// MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cmd
-
-import (
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"os"
-	"strconv"
-	"strings"
-
-	"github.com/minio/minio/cmd/crypto"
-)
-
-const (
-	// EnvKMSMasterKey is the environment variable used to specify
-	// a KMS master key used to protect SSE-S3 per-object keys.
-	// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
-	EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY"
-
-	// EnvAutoEncryption is the environment variable used to en/disable
-	// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
-	// requires a valid KMS configuration and turns any non-SSE-C
-	// request into an SSE-S3 request.
-	// If present EnvAutoEncryption must be either "on" or "off".
-	EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION"
-)
-
-const (
-	// EnvVaultEndpoint is the environment variable used to specify
-	// the vault HTTPS endpoint.
-	EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
-
-	// EnvVaultAuthType is the environment variable used to specify
-	// the authentication type for vault.
-	EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
-
-	// EnvVaultAppRoleID is the environment variable used to specify
-	// the vault AppRole ID.
-	EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
-
-	// EnvVaultAppSecretID is the environment variable used to specify
-	// the vault AppRole secret corresponding to the AppRole ID.
-	EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
-
-	// EnvVaultKeyVersion is the environment variable used to specify
-	// the vault key version.
-	EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
-
-	// EnvVaultKeyName is the environment variable used to specify
-	// the vault named key-ring. In the S3 context it's referred as
-	// customer master key ID (CMK-ID).
-	EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
-
-	// EnvVaultCAPath is the environment variable used to specify the
-	// path to a directory of PEM-encoded CA cert files. These CA cert
-	// files are used to authenticate MinIO to Vault over mTLS.
-	EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
-
-	// EnvVaultNamespace is the environment variable used to specify
-	// vault namespace. The vault namespace is used if the enterprise
-	// version of Hashicorp Vault is used.
-	EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
-)
-
-// Environment provides functions for accessing environment
-// variables.
-var Environment = environment{}
-
-type environment struct{}
-
-// Get retrieves the value of the environment variable named
-// by the key. If the variable is present in the environment the
-// value (which may be empty) is returned. Otherwise it returns
-// the specified default value.
-func (environment) Get(key, defaultValue string) string {
-	if v, ok := os.LookupEnv(key); ok {
-		return v
-	}
-	return defaultValue
-}
-
-// Lookup retrieves the value of the environment variable named
-// by the key. If the variable is present in the environment the
-// value (which may be empty) is returned and the boolean is true.
-// Otherwise the returned value will be empty and the boolean will
-// be false.
-func (environment) Lookup(key string) (string, bool) { return os.LookupEnv(key) }
-
-// LookupKMSConfig extracts the KMS configuration provided by environment
-// variables and merge them with the provided KMS configuration. The
-// merging follows the following rules:
-//
-// 1. A valid value provided as environment variable is higher prioritized
-// than the provided configuration and overwrites the value from the
-// configuration file.
-//
-// 2. A value specified as environment variable never changes the configuration
-// file. So it is never made a persistent setting.
-//
-// It sets the global KMS configuration according to the merged configuration
-// on success.
-func (env environment) LookupKMSConfig(config crypto.KMSConfig) (err error) {
-	// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
-	config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint)
-	config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath)
-	config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type)
-	config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID)
-	config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret)
-	config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name)
-	config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace)
-	keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version))
-	config.Vault.Key.Version, err = strconv.Atoi(keyVersion)
-	if err != nil {
-		return fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion)
-	}
-	if err = config.Vault.Verify(); err != nil {
-		return err
-	}
-
-	// Lookup KMS master keys - only available through ENV.
-	if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
-		if !config.Vault.IsEmpty() { // Vault and KMS master key provided
-			return errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
-		}
-		globalKMSKeyID, GlobalKMS, err = parseKMSMasterKey(masterKey)
-		if err != nil {
-			return err
-		}
-	}
-	if !config.Vault.IsEmpty() {
-		GlobalKMS, err = crypto.NewVault(config.Vault)
-		if err != nil {
-			return err
-		}
-		globalKMSKeyID = config.Vault.Key.Name
-	}
-
-	autoEncryption, err := ParseBoolFlag(env.Get(EnvAutoEncryption, "off"))
-	if err != nil {
-		return err
-	}
-	globalAutoEncryption = bool(autoEncryption)
-	if globalAutoEncryption && GlobalKMS == nil { // auto-encryption enabled but no KMS
-		return errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present")
-	}
-	return nil
-}
-
-// parseKMSMasterKey parses the value of the environment variable
-// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
-func parseKMSMasterKey(envArg string) (string, crypto.KMS, error) {
-	values := strings.SplitN(envArg, ":", 2)
-	if len(values) != 2 {
-		return "", nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
-	}
-	var (
-		keyID  = values[0]
-		hexKey = values[1]
-	)
-	if len(hexKey) != 64 { // 2 hex bytes = 1 byte
-		return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
-	}
-	var masterKey [32]byte
-	if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
-		return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
-	}
-	return keyID, crypto.NewKMS(masterKey), nil
-}

cmd/format-disk-cache.go

@@ -30,6 +30,7 @@ import (
 	"strings"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/sio"
 )
 
 const (
@@ -362,7 +363,7 @@ func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats [
 
 // reads cached object on disk and writes it back after adding bitrot
 // hashsum per block as per the new disk cache format.
-func migrateData(ctx context.Context, c *diskCache, oldfile, destDir string) error {
+func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile, destDir string, metadata map[string]string) error {
 	st, err := os.Stat(oldfile)
 	if err != nil {
 		err = osErrToFSFileErr(err)
@@ -372,7 +373,18 @@ func migrateData(ctx context.Context, c *diskCache, oldfile, destDir string) err
 	if err != nil {
 		return err
 	}
-	_, err = c.bitrotWriteToCache(ctx, destDir, readCloser, st.Size())
+	var reader io.Reader = readCloser
+
+	actualSize := uint64(st.Size())
+	if globalCacheKMS != nil {
+		reader, err = newCacheEncryptReader(readCloser, bucket, object, metadata)
+		if err != nil {
+			return err
+		}
+		actualSize, _ = sio.EncryptedSize(uint64(st.Size()))
+	}
+
+	_, err = c.bitrotWriteToCache(ctx, destDir, reader, uint64(actualSize))
 	return err
 }
 
@@ -401,6 +413,7 @@ func migrateOldCache(ctx context.Context, c *diskCache) error {
 	}
 
 	for _, bucket := range buckets {
+		bucket = strings.TrimSuffix(bucket, SlashSeparator)
 		var objMetaPaths []string
 		root := path.Join(oldCacheBucketsPath, bucket)
 		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
@@ -422,10 +435,23 @@ func migrateOldCache(ctx context.Context, c *diskCache) error {
 				return err
 			}
 			prevCachedPath := path.Join(c.dir, bucket, object)
+
+			// get old cached metadata
+			oldMetaPath := pathJoin(oldCacheBucketsPath, bucket, object, cacheMetaJSONFile)
+			metaPath := pathJoin(destdir, cacheMetaJSONFile)
+			metaBytes, err := ioutil.ReadFile(oldMetaPath)
+			if err != nil {
+				return err
+			}
+			// marshal cache metadata after adding version and stat info
+			meta := &cacheMeta{}
+			if err = json.Unmarshal(metaBytes, &meta); err != nil {
+				return err
+			}
 			// move cached object to new cache directory path
 			// migrate cache data and add bit-rot protection hash sum
 			// at the start of each block
-			if err := migrateData(ctx, c, prevCachedPath, destdir); err != nil {
+			if err := migrateCacheData(ctx, c, bucket, object, prevCachedPath, destdir, meta.Meta); err != nil {
 				continue
 			}
 			stat, err := os.Stat(prevCachedPath)
@@ -440,19 +466,7 @@ func migrateOldCache(ctx context.Context, c *diskCache) error {
 			if err := os.Remove(prevCachedPath); err != nil {
 				return err
 			}
-
 			// move cached metadata after changing cache metadata version
-			oldMetaPath := pathJoin(oldCacheBucketsPath, bucket, object, cacheMetaJSONFile)
-			metaPath := pathJoin(destdir, cacheMetaJSONFile)
-			metaBytes, err := ioutil.ReadFile(oldMetaPath)
-			if err != nil {
-				return err
-			}
-			// marshal cache metadata after adding version and stat info
-			meta := &cacheMeta{}
-			if err = json.Unmarshal(metaBytes, &meta); err != nil {
-				return err
-			}
 			meta.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
 			meta.Version = cacheMetaVersion
 			meta.Stat.Size = stat.Size()

cmd/format-fs.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"time"
 
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 )
@@ -153,7 +154,7 @@ func formatFSMigrate(ctx context.Context, wlk *lock.LockedFile, fsPath string) e
 		return err
 	}
 	if version != formatFSVersionV2 {
-		return uiErrUnexpectedBackendVersion(fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV2, version))
+		return config.ErrUnexpectedBackendVersion(fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV2, version))
 	}
 	return nil
 }

cmd/format-meta.go

@@ -20,9 +20,6 @@ package cmd
 const (
 	// Format config file carries backend format specific details.
 	formatConfigFile = "format.json"
-
-	// Format config tmp file carries backend format.
-	formatConfigFileTmp = "format.json.tmp"
 )
 
 const (

cmd/format-xl.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -291,16 +292,6 @@ func formatXLMigrateV2ToV3(export string) error {
 	return ioutil.WriteFile(formatPath, b, 0644)
 }
 
-// Returns true, if one of the errors is non-nil and is Unformatted disk.
-func hasAnyErrorsUnformatted(errs []error) bool {
-	for _, err := range errs {
-		if err != nil && err == errUnformattedDisk {
-			return true
-		}
-	}
-	return false
-}
-
 // countErrs - count a specific error.
 func countErrs(errs []error, err error) int {
 	var i = 0
@@ -325,7 +316,7 @@ func quorumUnformattedDisks(errs []error) bool {
 // loadFormatXLAll - load all format config from all input disks in parallel.
 func loadFormatXLAll(storageDisks []StorageAPI) ([]*formatXLV3, []error) {
 	// Initialize sync waitgroup.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	// Initialize list of errors.
 	var sErrs = make([]error, len(storageDisks))
@@ -367,16 +358,18 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
 		return err
 	}
 
+	tmpFormatJSON := mustGetUUID() + ".json"
+
 	// Purge any existing temporary file, okay to ignore errors here.
-	defer disk.DeleteFile(minioMetaBucket, formatConfigFileTmp)
+	defer disk.DeleteFile(minioMetaBucket, tmpFormatJSON)
 
 	// Append file `format.json.tmp`.
-	if err = disk.AppendFile(minioMetaBucket, formatConfigFileTmp, formatBytes); err != nil {
+	if err = disk.WriteAll(minioMetaBucket, tmpFormatJSON, bytes.NewReader(formatBytes)); err != nil {
 		return err
 	}
 
-	// Rename file `format.json.tmp` --> `format.json`.
-	return disk.RenameFile(minioMetaBucket, formatConfigFileTmp, minioMetaBucket, formatConfigFile)
+	// Rename file `uuid.json` --> `format.json`.
+	return disk.RenameFile(minioMetaBucket, tmpFormatJSON, minioMetaBucket, formatConfigFile)
 }
 
 var ignoredHiddenDirectories = []string{
@@ -652,7 +645,7 @@ func formatXLV3Check(reference *formatXLV3, format *formatXLV3) error {
 func saveFormatXLAll(ctx context.Context, storageDisks []StorageAPI, formats []*formatXLV3) error {
 	var errs = make([]error, len(storageDisks))
 
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	// Write `format.json` to all disks.
 	for index, disk := range storageDisks {
@@ -684,34 +677,22 @@ func closeStorageDisks(storageDisks []StorageAPI) {
 	}
 }
 
-// Initialize storage disks based on input arguments.
-func initStorageDisks(endpoints EndpointList) ([]StorageAPI, error) {
+// Initialize storage disks for each endpoint.
+// Errors are returned for each endpoint with matching index.
+func initStorageDisksWithErrors(endpoints EndpointList) ([]StorageAPI, []error) {
 	// Bootstrap disks.
 	storageDisks := make([]StorageAPI, len(endpoints))
+	errs := make([]error, len(endpoints))
+	var wg sync.WaitGroup
 	for index, endpoint := range endpoints {
-		storage, err := newStorageAPI(endpoint)
-		if err != nil && err != errDiskNotFound {
-			return nil, err
-		}
-		storageDisks[index] = storage
+		wg.Add(1)
+		go func(index int, endpoint Endpoint) {
+			defer wg.Done()
+			storageDisks[index], errs[index] = newStorageAPI(endpoint)
+		}(index, endpoint)
 	}
-	return storageDisks, nil
-}
-
-// Runs through the faulty disks and record their errors.
-func initDisksWithErrors(endpoints EndpointList) ([]StorageAPI, []error) {
-	storageDisks := make([]StorageAPI, len(endpoints))
-	var dErrs = make([]error, len(storageDisks))
-	for index, endpoint := range endpoints {
-		storage, err := newStorageAPI(endpoint)
-		if err != nil {
-			logger.LogIf(context.Background(), err)
-			dErrs[index] = err
-			continue
-		}
-		storageDisks[index] = storage
-	}
-	return storageDisks, dErrs
+	wg.Wait()
+	return storageDisks, errs
 }
 
 // formatXLV3ThisEmpty - find out if '.This' field is empty
@@ -812,7 +793,7 @@ func initFormatXLMetaVolume(storageDisks []StorageAPI, formats []*formatXLV3) er
 	// This happens for the first time, but keep this here since this
 	// is the only place where it can be made expensive optimizing all
 	// other calls. Create minio meta volume, if it doesn't exist yet.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	// Initialize errs to collect errors inside go-routine.
 	var errs = make([]error, len(storageDisks))

cmd/format-xl_test.go

@@ -85,9 +85,11 @@ func TestFixFormatV3(t *testing.T) {
 	}
 	endpoints := mustGetNewEndpointList(xlDirs...)
 
-	storageDisks, err := initStorageDisks(endpoints)
-	if err != nil {
-		t.Fatal(err)
+	storageDisks, errs := initStorageDisksWithErrors(endpoints)
+	for _, err := range errs {
+		if err != nil && err != errDiskNotFound {
+			t.Fatal(err)
+		}
 	}
 
 	format := newFormatXLV3(1, 8)
@@ -566,3 +568,36 @@ func TestNewFormatSets(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkInitStorageDisks256(b *testing.B) {
+	benchmarkInitStorageDisksN(b, 256)
+}
+
+func BenchmarkInitStorageDisks1024(b *testing.B) {
+	benchmarkInitStorageDisksN(b, 1024)
+}
+
+func BenchmarkInitStorageDisks2048(b *testing.B) {
+	benchmarkInitStorageDisksN(b, 2048)
+}
+
+func BenchmarkInitStorageDisksMax(b *testing.B) {
+	benchmarkInitStorageDisksN(b, 32*204)
+}
+
+func benchmarkInitStorageDisksN(b *testing.B, nDisks int) {
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		b.Fatal(err)
+	}
+	endpoints := mustGetNewEndpointList(fsDirs...)
+	b.RunParallel(func(pb *testing.PB) {
+		endpoints := endpoints
+		for pb.Next() {
+			initStorageDisksWithErrors(endpoints)
+		}
+	})
+}

cmd/fs-v1-metadata.go

@@ -27,10 +27,11 @@ import (
 	pathutil "path"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/mimedb"
-	"github.com/tidwall/gjson"
 )
 
 // FS format, and object metadata.
@@ -166,7 +167,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	objInfo.ETag = extractETag(m.Meta)
 	objInfo.ContentType = m.Meta["content-type"]
 	objInfo.ContentEncoding = m.Meta["content-encoding"]
-	if storageClass, ok := m.Meta[amzStorageClass]; ok {
+	if storageClass, ok := m.Meta[xhttp.AmzStorageClass]; ok {
 		objInfo.StorageClass = storageClass
 	} else {
 		objInfo.StorageClass = globalMinioDefaultStorageClass
@@ -203,44 +204,6 @@ func (m *fsMetaV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
 	return fi.Size(), nil
 }
 
-func parseFSVersion(fsMetaBuf []byte) string {
-	return gjson.GetBytes(fsMetaBuf, "version").String()
-}
-
-func parseFSMetaMap(fsMetaBuf []byte) map[string]string {
-	// Get xlMetaV1.Meta map.
-	metaMapResult := gjson.GetBytes(fsMetaBuf, "meta").Map()
-	metaMap := make(map[string]string)
-	for key, valResult := range metaMapResult {
-		metaMap[key] = valResult.String()
-	}
-	return metaMap
-}
-
-func parseFSPartsArray(fsMetaBuf []byte) []ObjectPartInfo {
-	// Get xlMetaV1.Parts array
-	var partsArray []ObjectPartInfo
-
-	partsArrayResult := gjson.GetBytes(fsMetaBuf, "parts")
-	partsArrayResult.ForEach(func(key, part gjson.Result) bool {
-		partJSON := part.String()
-		number := gjson.Get(partJSON, "number").Int()
-		name := gjson.Get(partJSON, "name").String()
-		etag := gjson.Get(partJSON, "etag").String()
-		size := gjson.Get(partJSON, "size").Int()
-		actualSize := gjson.Get(partJSON, "actualSize").Int()
-		partsArray = append(partsArray, ObjectPartInfo{
-			Number:     int(number),
-			Name:       name,
-			ETag:       etag,
-			Size:       size,
-			ActualSize: int64(actualSize),
-		})
-		return true
-	})
-	return partsArray
-}
-
 func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64, err error) {
 	var fsMetaBuf []byte
 	fi, err := lk.Stat()
@@ -260,8 +223,10 @@ func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64,
 		return 0, io.EOF
 	}
 
-	// obtain version.
-	m.Version = parseFSVersion(fsMetaBuf)
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	if err = json.Unmarshal(fsMetaBuf, m); err != nil {
+		return 0, err
+	}
 
 	// Verify if the format is valid, return corrupted format
 	// for unrecognized formats.
@@ -271,12 +236,6 @@ func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64,
 		return 0, errCorruptedFormat
 	}
 
-	// obtain parts information
-	m.Parts = parseFSPartsArray(fsMetaBuf)
-
-	// obtain metadata.
-	m.Meta = parseFSMetaMap(fsMetaBuf)
-
 	// Success.
 	return int64(len(fsMetaBuf)), nil
 }

cmd/fs-v1-multipart.go

@@ -28,6 +28,7 @@ import (
 	"strings"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
 	mioutil "github.com/minio/minio/pkg/ioutil"
 )
@@ -456,7 +457,8 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 	}
 	for i, part := range result.Parts {
 		var stat os.FileInfo
-		stat, err = fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag, part.ActualSize)))
+		stat, err = fsStatFile(ctx, pathJoin(uploadIDDir,
+			fs.encodePartFile(part.PartNumber, part.ETag, part.ActualSize)))
 		if err != nil {
 			return result, toObjectErr(err)
 		}
@@ -470,7 +472,13 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 		return result, err
 	}
 
-	result.UserDefined = parseFSMetaMap(fsMetaBytes)
+	var fsMeta fsMetaV1
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	if err = json.Unmarshal(fsMetaBytes, &fsMeta); err != nil {
+		return result, err
+	}
+
+	result.UserDefined = fsMeta.Meta
 	return result, nil
 }
 

cmd/fs-v1-multipart_test.go

@@ -95,7 +95,6 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 
-	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 	objectName := "object"
 	data := []byte("12345")
@@ -105,7 +104,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -113,10 +112,13 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	md5Hex := getMD5Hash(data)
 	sha256sum := ""
 
-	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	_, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{})
-	if !isSameType(err, BucketNotFound{}) {
-		t.Fatal("Unexpected error ", err)
+	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(newDisk)
+	obj = initFSObjects(newDisk, t)
+	if _, err = obj.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{}); err != nil {
+		if !isSameType(err, BucketNotFound{}) {
+			t.Fatal("Unexpected error ", err)
+		}
 	}
 }
 
@@ -127,7 +129,6 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 
-	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 	objectName := "object"
 	data := []byte("12345")
@@ -136,7 +137,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -144,8 +145,10 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	md5Hex := getMD5Hash(data)
 
 	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
-	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
+	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(newDisk)
+	obj = initFSObjects(newDisk, t)
+	if _, err := obj.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
@@ -159,7 +162,6 @@ func TestCompleteMultipartUpload(t *testing.T) {
 	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 
-	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 	objectName := "object"
 	data := []byte("12345")
@@ -168,20 +170,19 @@ func TestCompleteMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
+	if _, err := obj.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
-
-	if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
+	if _, err := obj.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 }
@@ -193,7 +194,6 @@ func TestAbortMultipartUpload(t *testing.T) {
 	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 
-	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 	objectName := "object"
 	data := []byte("12345")
@@ -202,18 +202,18 @@ func TestAbortMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
+	if _, err := obj.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 	time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
-	if err := fs.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
+	if err := obj.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 }
@@ -226,7 +226,6 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 
 	obj := initFSObjects(disk, t)
 
-	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 	objectName := "object"
 
@@ -234,13 +233,15 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	_, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	_, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
-	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	if _, err := fs.ListMultipartUploads(context.Background(), bucketName, objectName, "", "", "", 1000); err != nil {
+	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(newDisk)
+	obj = initFSObjects(newDisk, t)
+	if _, err := obj.ListMultipartUploads(context.Background(), bucketName, objectName, "", "", "", 1000); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}

cmd/fs-v1.go

@@ -30,7 +30,9 @@ import (
 	"sync/atomic"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v6/pkg/s3utils"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/lock"
@@ -115,7 +117,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		if err == errMinDiskSize {
 			return nil, err
 		}
-		return nil, uiErrUnableToWriteInBackend(err)
+		return nil, config.ErrUnableToWriteInBackend(err)
 	}
 
 	// Assign a new UUID for FS minio mode. Each server instance
@@ -1092,13 +1094,19 @@ func (fs *FSObjects) getObjectETag(ctx context.Context, bucket, entry string, lo
 		return "", toObjectErr(err, bucket, entry)
 	}
 
+	var fsMeta fsMetaV1
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	if err = json.Unmarshal(fsMetaBuf, &fsMeta); err != nil {
+		return "", err
+	}
+
 	// Check if FS metadata is valid, if not return error.
-	if !isFSMetaValid(parseFSVersion(fsMetaBuf)) {
+	if !isFSMetaValid(fsMeta.Version) {
 		logger.LogIf(ctx, errCorruptedFormat)
 		return "", toObjectErr(errCorruptedFormat, bucket, entry)
 	}
 
-	return extractETag(parseFSMetaMap(fsMetaBuf)), nil
+	return extractETag(fsMeta.Meta), nil
 }
 
 // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool

cmd/gateway-common.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,12 +18,14 @@ package cmd
 
 import (
 	"net/http"
-	"os"
 	"strings"
 
+	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/hash"
+	xnet "github.com/minio/minio/pkg/net"
 
 	minio "github.com/minio/minio-go/v6"
 )
@@ -299,7 +301,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		object = params[1]
 	}
 
-	if isNetworkOrHostDown(err) {
+	if xnet.IsNetworkOrHostDown(err) {
 		return BackendDown{}
 	}
 
@@ -364,14 +366,14 @@ func parseGatewaySSE(s string) (gatewaySSE, error) {
 			gwSlice = append(gwSlice, v)
 			continue
 		}
-		return nil, uiErrInvalidGWSSEValue(nil).Msg("gateway SSE cannot be (%s) ", v)
+		return nil, config.ErrInvalidGWSSEValue(nil).Msg("gateway SSE cannot be (%s) ", v)
 	}
 	return gatewaySSE(gwSlice), nil
 }
 
 // handle gateway env vars
 func handleGatewayEnvVars() {
-	gwsseVal, ok := os.LookupEnv("MINIO_GATEWAY_SSE")
+	gwsseVal, ok := env.Lookup("MINIO_GATEWAY_SSE")
 	if ok {
 		var err error
 		GlobalGatewaySSE, err = parseGatewaySSE(gwsseVal)