s3: Fix precondition failed in CopyObjectPart when src is encrypted (#7276)

CopyObject precondition checks into GetObjectReader
in order to perform SSE-C pre-condition checks using the
last 32 bytes of encrypted ETag rather than the decrypted
ETag

This also necessitates moving precondition checks for
gateways to gateway layer rather than object handler check

.github/ISSUE_TEMPLATE.md

@@ -24,6 +24,10 @@
 <!--- How has this issue affected you? What are you trying to accomplish? -->
 <!--- Providing context helps us come up with a solution that is most useful in the real world -->
 
+## Regression
+<!-- Is this issue a regression? (Yes / No) -->
+<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
+
 ## Your Environment
 <!--- Include as many relevant details about the environment you experienced the bug in -->
 * Version used (`minio version`):

.github/PULL_REQUEST_TEMPLATE.md

@@ -7,6 +7,10 @@
 <!--- Why is this change required? What problem does it solve? -->
 <!--- If it fixes an open issue, please link to the issue here. -->
 
+## Regression
+<!-- Is this PR fixing a regression? (Yes / No) -->
+<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
+
 ## How Has This Been Tested?
 <!--- Please describe in detail how you tested your changes. -->
 <!--- Include details of your testing environment, and the tests you ran to -->

.gitignore

@@ -22,3 +22,5 @@ parts/
 prime/
 stage/
 .sia_temp/
+config.json
+healthcheck

.travis.yml

@@ -1,34 +1,50 @@
 go_import_path: github.com/minio/minio
-sudo: required
-
-services:
- - docker
-
-dist: trusty
 
 language: go
 
-os:
-- linux
+# this ensures PRs based on a local branch are not built twice
+# the downside is that a PR targeting a different branch is not built
+# but as a workaround you can add the branch to this list
+branches:
+  only:
+    - master
 
-env:
-- ARCH=x86_64
+matrix:
+  include:
+    - os: linux
+      dist: trusty
+      sudo: required
+      env:
+        - ARCH=x86_64
+        - CGO_ENABLED=0
+      go: 1.11.4
+      script:
+        - make
+        - diff -au <(gofmt -s -d cmd) <(printf "")
+        - diff -au <(gofmt -s -d pkg) <(printf "")
+        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 15m "$d"; done
+        - make verifiers
+        - make crosscompile
+        - make verify
+        - make coverage
+        - cd browser && yarn && yarn test && cd ..
+    - os: windows
+      env:
+        - ARCH=x86_64
+        - CGO_ENABLED=0
+      go: 1.11.4
+      script:
+        - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
+        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done
+        - bash buildscripts/go-coverage.sh
+
+before_script:
+  # Add an IPv6 config - see the corresponding Travis issue
+  # https://github.com/travis-ci/travis-ci/issues/8361
+  - if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi
 
 before_install:
-- nvm install stable
-
-script:
-## Run all the tests
-- make
-- diff -au <(gofmt -s -d cmd) <(printf "")
-- diff -au <(gofmt -s -d pkg) <(printf "")
-- make test GOFLAGS="-timeout 15m -race -v"
-- make coverage
-- node --version
-- cd browser && yarn && yarn test && cd ..
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
 
 after_success:
 - bash <(curl -s https://codecov.io/bash)
-
-go:
-- '1.10.1'

CONTRIBUTING.md

@@ -6,7 +6,7 @@
 
 Start by forking the Minio GitHub repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details:
 
-### Setup your Minio Github Repository
+### Setup your Minio GitHub Repository
 Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL of your Minio fork (you will need it for the `git clone` command below).
 
 ```sh
@@ -68,4 +68,4 @@ To remove a dependency
 - Run `make pkg-remove PKG=foo/bar` from top-level directory
 
 ### What are the coding guidelines for Minio?
-``Minio`` is fully conformant with Golang style. Refer: [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project. If you observe offending code, please feel free to send a pull request or ping us on [Slack](slack.minio.io).
+``Minio`` is fully conformant with Golang style. Refer: [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project. If you observe offending code, please feel free to send a pull request or ping us on [Slack](https://slack.minio.io).

Dockerfile

@@ -1,34 +1,39 @@
-FROM golang:1.10.1-alpine3.7
+FROM golang:1.11.4-alpine3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
 ENV GOPATH /go
-ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
+
+WORKDIR /go/src/github.com/minio/
+
+RUN  \
+     apk add --no-cache git && \
+     go get -v -d github.com/minio/minio && \
+     cd /go/src/github.com/minio/minio && \
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
+     go build -ldflags "-s -w" -o /usr/bin/healthcheck dockerscripts/healthcheck.go 
+
+FROM alpine:3.7
+
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
-WORKDIR /go/src/github.com/minio/
+EXPOSE 9000
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY --from=0 /go/bin/minio /usr/bin/minio
+COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
-     apk add --no-cache ca-certificates curl && \
-     apk add --no-cache --virtual .build-deps git && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     go get -v -d github.com/minio/minio && \
-     cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
-
-EXPOSE 9000
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.dev

@@ -1,27 +1,20 @@
-FROM golang:1.10.1-alpine3.7
+FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
-ENV GOPATH /go
-ENV PATH $PATH:$GOPATH/bin
-ENV CGO_ENABLED 0
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck /usr/bin/
+COPY minio /usr/bin/
+
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
-WORKDIR /go/src/github.com/minio/
-
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
-
-COPY . /go/src/github.com/minio/minio
-
-RUN  \
-     apk add --no-cache ca-certificates curl && \
-     apk add --no-cache --virtual .build-deps git && \
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 
@@ -29,7 +22,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.release

@@ -1,21 +1,34 @@
+FROM golang:1.11.4-alpine3.7
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+WORKDIR /go/src/github.com/minio/
+
+RUN  \
+     apk add --no-cache git && \
+     go get -v -d github.com/minio/minio && \
+     cd /go/src/github.com/minio/minio/dockerscripts && \
+     go build -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
+
 FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
 RUN \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps curl && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck.sh
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 
@@ -23,7 +36,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.simpleci

@@ -0,0 +1,72 @@
+#-------------------------------------------------------------
+# Stage 1: Build and Unit tests
+#-------------------------------------------------------------
+FROM golang:1.11.4
+
+COPY . /go/src/github.com/minio/minio
+WORKDIR /go/src/github.com/minio/minio
+
+RUN apt-get update && apt-get install -y jq
+
+RUN make
+RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
+RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
+
+RUN apt-get update && \
+      apt-get -y install sudo
+RUN touch  /etc/sudoers
+
+RUN echo "root ALL=(ALL) ALL" >> /etc/sudoers
+RUN echo "ci ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+RUN echo "Defaults    env_reset" >> /etc/sudoers
+RUN echo 'Defaults    secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go:/usr/local/go/bin"' >> /etc/sudoers
+
+RUN mkdir -p /home/ci/.cache
+
+RUN groupadd -g 999 ci && \
+    useradd -r -u 999 -g ci ci
+
+RUN chown -R ci:ci /go
+RUN chown -R ci:ci /home/ci
+
+USER ci
+
+RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 15m "$d"; done
+RUN make verifiers
+RUN make crosscompile
+RUN make coverage
+RUN make verify
+
+#-------------------------------------------------------------
+# Stage 2: Test Frontend
+#-------------------------------------------------------------
+FROM node:10.15-stretch-slim
+
+COPY browser /minio/browser
+WORKDIR /minio/browser
+
+RUN yarn
+RUN yarn test
+
+#-------------------------------------------------------------
+# Stage 3: Run Gateway Tests
+#-------------------------------------------------------------
+
+FROM ubuntu:16.04
+
+COPY --from=0 /go/src/github.com/minio/minio/minio ./minio
+COPY buildscripts/gateway-tests.sh ./gateway-tests.sh
+RUN apt-get update && apt-get install -y git wget jq curl dnsmasq
+
+RUN wget https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz && \
+	tar -C /usr/local -xzf go1.11.4.linux-amd64.tar.gz
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG C.UTF-8
+ENV GOROOT /usr/local/go
+
+RUN mkdir -p /go
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+
+RUN ./gateway-tests.sh

MAINTAINERS.md

@@ -1,6 +1,6 @@
 # For maintainers only
 
-### Setup your minio Github Repository
+### Setup your minio GitHub Repository
 
 Fork [minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository.
 ```bash

Makefile

@@ -13,52 +13,48 @@ checks:
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
 getdeps:
-	@echo "Installing golint" && go get -u github.com/golang/lint/golint
-	@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
-	@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
+	@echo "Installing golint" && go get -u golang.org/x/lint/golint
+	@echo "Installing staticcheck" && go get -u honnef.co/go/tools/...
 	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
-	@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
 
-verifiers: getdeps vet fmt lint cyclo deadcode spelling
+crosscompile:
+	@(env bash $(PWD)/buildscripts/cross-compile.sh)
+
+verifiers: getdeps vet fmt lint staticcheck spelling
 
 vet:
 	@echo "Running $@"
-	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult cmd
-	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult pkg
+	@go vet github.com/minio/minio/...
 
 fmt:
 	@echo "Running $@"
-	@gofmt -d cmd
-	@gofmt -d pkg
+	@gofmt -d cmd/
+	@gofmt -d pkg/
 
 lint:
 	@echo "Running $@"
-	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd...
-	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg...
+	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
+	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
 
-ineffassign:
+staticcheck:
 	@echo "Running $@"
-	@${GOPATH}/bin/ineffassign .
-
-cyclo:
-	@echo "Running $@"
-	@${GOPATH}/bin/gocyclo -over 100 cmd
-	@${GOPATH}/bin/gocyclo -over 100 pkg
-
-deadcode:
-	@echo "Running $@"
-	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
+	@${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
+	@${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
 
 spelling:
-	@${GOPATH}/bin/misspell -error `find cmd/`
-	@${GOPATH}/bin/misspell -error `find pkg/`
-	@${GOPATH}/bin/misspell -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find cmd/`
+	@${GOPATH}/bin/misspell -locale US -error `find pkg/`
+	@${GOPATH}/bin/misspell -locale US -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
+	@${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build
 	@echo "Running unit tests"
-	@go test $(GOFLAGS) ./...
+	@CGO_ENABLED=0 go test -tags kqueue ./...
+
+verify: build
 	@echo "Verifying build"
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
@@ -69,7 +65,11 @@ coverage: build
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@CGO_ENABLED=0 go build --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
+	@GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
+	@GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags="-s -w" -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go
+
+docker: build
+	@docker build -t $(TAG) . -f Dockerfile.dev
 
 pkg-add:
 	@echo "Adding new package $(PKG)"
@@ -89,12 +89,13 @@ pkg-list:
 # Builds minio and installs it to $GOPATH/bin.
 install: build
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
-	@cp $(PWD)/minio $(GOPATH)/bin/minio
+	@mkdir -p $(GOPATH)/bin && cp $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
 clean:
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
+	@find . -name '*~' | xargs rm -fv
 	@rm -rvf minio
 	@rm -rvf build
 	@rm -rvf release

README.md

@@ -17,7 +17,7 @@ docker run -p 9000:9000 minio/minio server /data
 docker pull minio/minio:edge
 docker run -p 9000:9000 minio/minio:edge server /data
 ```
-Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
+Note: Docker will not display the autogenerated keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use autogenerated keys with containers. Please visit Minio Docker quickstart guide for more information [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
 ### Homebrew
@@ -53,6 +53,15 @@ chmod +x minio
 ./minio server /data
 ```
 
+| Platform| Architecture | URL|
+| ----------| -------- | ------|
+|GNU/Linux|ppc64le|https://dl.minio.io/server/minio/release/linux-ppc64le/minio |
+```sh
+wget https://dl.minio.io/server/minio/release/linux-ppc64le/minio
+chmod +x minio
+./minio server /data
+```
+
 ## Microsoft Windows
 ### Binary Download
 | Platform| Architecture | URL|

appveyor.yml

@@ -1,55 +0,0 @@
-# version format
-version: "{build}"
-
-# Operating system (build VM template)
-os: Windows Server 2012 R2
-
-# Platform.
-platform: x64
-
-clone_folder: c:\gopath\src\github.com\minio\minio
-
-# Environment variables
-environment:
-  GOPATH: c:\gopath
-  GOROOT: c:\go
-
-# scripts that run after cloning repository
-install:
-  - set PATH=%GOPATH%\bin;%GOROOT%\bin;%PATH%
-  - go version
-  - go env
-  - python --version
-
-# To run your custom scripts instead of automatic MSBuild
-build_script:
-  # Compile
-  # We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
-  - ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
-  - appveyor AddCompilationMessage "Starting Compile"
-  - cd c:\gopath\src\github.com\minio\minio
-  - go run buildscripts/gen-ldflags.go > temp.txt
-  - set /p BUILD_LDFLAGS=<temp.txt
-  - go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
-  - appveyor AddCompilationMessage "Compile Success"
-
-# To run your custom scripts instead of automatic tests
-test_script:
-  # Unit tests
-  - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
-  - mkdir build\coverage
-  - for /f "" %%G in ('go list github.com/minio/minio/... ^| find /i /v "browser/"') do ( go test -v -timeout 20m -race %%G )
-  - go test -v -timeout 20m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
-  - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
-
-after_test:
-  - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
-  - ps: Push-AppveyorArtifact build\coverage\coverage.txt
-  - ps: Push-AppveyorArtifact build\coverage\coverage.html
-  # Upload coverage report.
-  - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
-  - pip install codecov
-  - codecov -X gcov -f "build\coverage\coverage.txt"
-
-# to disable deployment
-deploy: off

browser/app/index.html

@@ -5,13 +5,13 @@
         <meta charset="UTF-8">
         <meta name="viewport" content="width=device-width, initial-scale=1">
         <title>Minio Browser</title>
-        <link rel="stylesheet" href="loader.css" type="text/css">
+        <link rel="stylesheet" href="/minio/loader.css" type="text/css">
     </head>
 
     <body>
         <div class="page-load">
             <div class="pl-inner">
-                <img src="logo.svg" alt="">
+                <img src="/minio/logo.svg" alt="">
             </div>
         </div>
         <div id="root"></div>
@@ -51,6 +51,6 @@
         <![endif]-->
 
         <script>currentUiVersion = 'MINIO_UI_VERSION'</script>
-        <script src="index_bundle.js"></script>
+        <script src="/minio/index_bundle.js"></script>
     </body>
 </html>

browser/app/js/alert/actions.js

@@ -22,7 +22,7 @@ export let alertId = 0
 export const set = alert => {
   const id = alertId++
   return (dispatch, getState) => {
-    if (alert.type !== "danger") {
+    if (alert.type !== "danger" || alert.autoClear) {
       setTimeout(() => {
         dispatch({
           type: CLEAR,

browser/app/js/browser/BrowserDropdown.js

@@ -89,7 +89,7 @@ export class BrowserDropdown extends React.Component {
           <Dropdown.Menu className="dropdown-menu-right">
             <li>
               <a target="_blank" href="https://github.com/minio/minio">
-                Github <i className="fa fa-github" />
+                GitHub <i className="fa fa-github" />
               </a>
             </li>
             <li>

browser/app/js/browser/ChangePasswordModal.js

@@ -42,7 +42,7 @@ export class ChangePasswordModal extends React.Component {
     const { serverInfo } = this.props
 
     // Check environment variables first.
-    if (serverInfo.info.isEnvCreds) {
+    if (serverInfo.info.isEnvCreds || serverInfo.info.isWorm) {
       this.setState({
         accessKey: "xxxxxxxxx",
         secretKey: "xxxxxxxxx",

browser/app/js/browser/Login.js

@@ -25,14 +25,35 @@ import web from "../web"
 import { Redirect } from "react-router-dom"
 
 export class Login extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      accessKey: "",
+      secretKey: ""
+    }
+  }
+
+  // Handle field changes
+  accessKeyChange(e) {
+    this.setState({
+      accessKey: e.target.value
+    })
+  }
+
+  secretKeyChange(e) {
+    this.setState({
+      secretKey: e.target.value
+    })
+  }
+
   handleSubmit(event) {
     event.preventDefault()
     const { showAlert, history } = this.props
     let message = ""
-    if (!document.getElementById("accessKey").value) {
+    if (this.state.accessKey === "") {
       message = "Access Key cannot be empty"
     }
-    if (!document.getElementById("secretKey").value) {
+    if (this.state.secretKey === "") {
       message = "Secret Key cannot be empty"
     }
     if (message) {
@@ -41,8 +62,8 @@ export class Login extends React.Component {
     }
     web
       .Login({
-        username: document.getElementById("accessKey").value,
-        password: document.getElementById("secretKey").value
+        username: this.state.accessKey,
+        password: this.state.secretKey
       })
       .then(res => {
         history.push("/")
@@ -77,6 +98,8 @@ export class Login extends React.Component {
         <div className="l-wrap">
           <form onSubmit={this.handleSubmit.bind(this)}>
             <InputGroup
+              value={this.state.accessKey}
+              onChange={this.accessKeyChange.bind(this)}
               className="ig-dark"
               label="Access Key"
               id="accessKey"
@@ -87,6 +110,8 @@ export class Login extends React.Component {
               autoComplete="username"
             />
             <InputGroup
+              value={this.state.secretKey}
+              onChange={this.secretKeyChange.bind(this)}
               className="ig-dark"
               label="Secret Key"
               id="secretKey"

browser/app/js/browser/StorageInfo.js

@@ -25,23 +25,16 @@ export class StorageInfo extends React.Component {
     fetchStorageInfo()
   }
   render() {
-    const { total, free } = this.props.storageInfo
-    const used = total - free
-    const usedPercent = used / total * 100 + "%"
-    const freePercent = free * 100 / total
+    const { used } = this.props.storageInfo
     return (
-      <div className="feh-usage">
+      <div className="feh-used">
         <div className="fehu-chart">
-          <div style={{ width: usedPercent }} />
+          <div style={{ width: 0 }} />
         </div>
         <ul>
           <li>
             <span>Used: </span>
-            {humanize.filesize(total - free)}
-          </li>
-          <li className="pull-right">
-            <span>Free: </span>
-            {humanize.filesize(total - used)}
+            {humanize.filesize(used)}
           </li>
         </ul>
       </div>

browser/app/js/browser/__tests__/Login.test.js

@@ -60,20 +60,25 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
     // case where both keys are empty - displays the second warning
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
 
     // case where access key is empty
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Access Key cannot be empty")
 
     // case where secret key is empty
-    document.getElementById("accessKey").value = "accessKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: ""
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
   })
@@ -85,11 +90,12 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
-    document.getElementById("accessKey").value = "accessKey"
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(web.Login).toHaveBeenCalledWith({
       "username": "accessKey", 

browser/app/js/browser/__tests__/StorageInfo.test.js

@@ -22,7 +22,7 @@ describe("StorageInfo", () => {
   it("should render without crashing", () => {
     shallow(
       <StorageInfo
-        storageInfo={{ total: 100, free: 60 }}
+        storageInfo={{ used: 60 }}
         fetchStorageInfo={jest.fn()}
       />
     )
@@ -32,7 +32,7 @@ describe("StorageInfo", () => {
     const fetchStorageInfo = jest.fn()
     shallow(
       <StorageInfo
-        storageInfo={{ total: 100, free: 60 }}
+        storageInfo={{ used: 60 }}
         fetchStorageInfo={fetchStorageInfo}
       />
     )

browser/app/js/browser/__tests__/actions.test.js

@@ -20,7 +20,7 @@ import * as actionsCommon from "../actions"
 
 jest.mock("../../web", () => ({
   StorageInfo: jest.fn(() => {
-    return Promise.resolve({ storageInfo: { Total: 100, Free: 60 } })
+    return Promise.resolve({ storageInfo: { Used: 60 } })
   }),
   ServerInfo: jest.fn(() => {
     return Promise.resolve({
@@ -40,7 +40,7 @@ describe("Common actions", () => {
   it("creates common/SET_STORAGE_INFO after fetching the storage details ", () => {
     const store = mockStore()
     const expectedActions = [
-      { type: "common/SET_STORAGE_INFO", storageInfo: { total: 100, free: 60 } }
+      { type: "common/SET_STORAGE_INFO", storageInfo: { used: 60 } }
     ]
     return store.dispatch(actionsCommon.fetchStorageInfo()).then(() => {
       const actions = store.getActions()

browser/app/js/browser/actions.js

@@ -34,7 +34,7 @@ export const fetchStorageInfo = () => {
     return web.StorageInfo().then(res => {
       const storageInfo = {
         total: res.storageInfo.Total,
-        free: res.storageInfo.Free
+        used:  res.storageInfo.Used
       }
       dispatch(setStorageInfo(storageInfo))
     })

browser/app/js/buckets/BucketList.js

@@ -67,8 +67,12 @@ const mapDispatchToProps = dispatch => {
   return {
     fetchBuckets: () => dispatch(actionsBuckets.fetchBuckets()),
     setBucketList: buckets => dispatch(actionsBuckets.setList(buckets)),
-    selectBucket: bucket => dispatch(actionsBuckets.selectBucket(bucket))
+    selectBucket: (bucket, prefix) =>
+      dispatch(actionsBuckets.selectBucket(bucket, prefix))
   }
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(BucketList)
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(BucketList)

browser/app/js/buckets/Policy.js

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { READ_ONLY, WRITE_ONLY, READ_WRITE } from '../constants'
+import { READ_ONLY, WRITE_ONLY, READ_WRITE, NONE } from '../constants'
 
 import React from "react"
 import { connect } from "react-redux"
@@ -42,37 +42,40 @@ export class Policy extends React.Component {
   render() {
     const {policy, prefix} = this.props
     let newPrefix = prefix
-
     if (newPrefix === '')
       newPrefix = '*'
 
-    return (
-      <div className="pmb-list">
-        <div className="pmbl-item">
-          { newPrefix }
+    if (policy === NONE) {
+      return <noscript />
+    } else {
+      return (
+        <div className="pmb-list">
+          <div className="pmbl-item">
+            { newPrefix }
+          </div>
+          <div className="pmbl-item">
+            <select className="form-control"
+              disabled
+              value={ policy }>
+              <option value={ READ_ONLY }>
+                Read Only
+              </option>
+              <option value={ WRITE_ONLY }>
+                Write Only
+              </option>
+              <option value={ READ_WRITE }>
+                Read and Write
+              </option>
+            </select>
+          </div>
+          <div className="pmbl-item">
+            <button className="btn btn-block btn-danger" onClick={ this.removePolicy.bind(this) }>
+              Remove
+            </button>
+          </div>
         </div>
-        <div className="pmbl-item">
-          <select className="form-control"
-            disabled
-            value={ policy }>
-            <option value={ READ_ONLY }>
-              Read Only
-            </option>
-            <option value={ WRITE_ONLY }>
-              Write Only
-            </option>
-            <option value={ READ_WRITE }>
-              Read and Write
-            </option>
-          </select>
-        </div>
-        <div className="pmbl-item">
-          <button className="btn btn-block btn-danger" onClick={ this.removePolicy.bind(this) }>
-            Remove
-          </button>
-        </div>
-      </div>
-    )
+      )
+    }
   }
 }
 

browser/app/js/buckets/__tests__/Policy.test.js

@@ -17,7 +17,7 @@
 import React from "react"
 import { shallow, mount } from "enzyme"
 import { Policy } from "../Policy"
-import { READ_ONLY, WRITE_ONLY, READ_WRITE } from "../../constants"
+import { READ_ONLY, WRITE_ONLY, READ_WRITE, NONE } from "../../constants"
 import web from "../../web"
 
 jest.mock("../../web", () => ({
@@ -31,6 +31,11 @@ describe("Policy", () => {
     shallow(<Policy currentBucket={"bucket"} prefix={"foo"} policy={READ_ONLY} />)
   })
 
+  it("should not render when policy is listed as 'none'", () => {
+    const wrapper = shallow(<Policy currentBucket={"bucket"} prefix={"foo"} policy={NONE} />)
+    expect(wrapper.find(".pmb-list").length).toBe(0)
+  })
+
   it("should call web.setBucketPolicy and fetchPolicies on submit", () => {
     const fetchPolicies = jest.fn()
     const wrapper = shallow(

browser/app/js/constants.js

@@ -23,6 +23,7 @@ export const minioBrowserPrefix = p.slice(0, p.indexOf("/", 1))
 export const READ_ONLY = "readonly"
 export const WRITE_ONLY = "writeonly"
 export const READ_WRITE = "readwrite"
+export const NONE = "none"
 
 export const SHARE_OBJECT_EXPIRY_DAYS = 5
 export const SHARE_OBJECT_EXPIRY_HOURS = 0

browser/app/js/objects/ObjectActions.js

@@ -57,7 +57,7 @@ export class ObjectActions extends React.Component {
     })
   }
   render() {
-    const { object, showShareObjectModal } = this.props
+    const { object, showShareObjectModal, shareObjectName } = this.props
     return (
       <Dropdown id={`obj-actions-${object.name}`}>
         <Dropdown.Toggle noCaret className="fia-toggle" />
@@ -67,7 +67,7 @@ export class ObjectActions extends React.Component {
             className="fiad-action"
             onClick={this.shareObject.bind(this)}
           >
-            <i className="fa fa-copy" />
+            <i className="fa fa-share-alt" />
           </a>
           <a
             href=""
@@ -77,7 +77,8 @@ export class ObjectActions extends React.Component {
             <i className="fa fa-trash" />
           </a>
         </Dropdown.Menu>
-        {showShareObjectModal && <ShareObjectModal object={object} />}
+        {(showShareObjectModal && shareObjectName === object.name) &&
+          <ShareObjectModal object={object} />}
         {this.state.showDeleteConfirmation && (
           <DeleteObjectConfirmModal
             deleteObject={this.deleteObject.bind(this)}
@@ -92,7 +93,8 @@ export class ObjectActions extends React.Component {
 const mapStateToProps = (state, ownProps) => {
   return {
     object: ownProps.object,
-    showShareObjectModal: state.objects.shareObject.show
+    showShareObjectModal: state.objects.shareObject.show,
+    shareObjectName: state.objects.shareObject.object
   }
 }
 

browser/app/js/objects/ObjectContainer.js

@@ -37,7 +37,7 @@ export const ObjectContainer = ({
   if (checkedObjectsCount == 0) {
     props.actionButtons = <ObjectActions object={object} />
   }
-  return <ObjectItem {...props} onClick={() => downloadObject(object.name)} />
+  return <ObjectItem {...props} />
 }
 
 const mapStateToProps = state => {
@@ -46,10 +46,4 @@ const mapStateToProps = state => {
   }
 }
 
-const mapDispatchToProps = dispatch => {
-  return {
-    downloadObject: object => dispatch(actionsObjects.downloadObject(object))
-  }
-}
-
-export default connect(mapStateToProps, mapDispatchToProps)(ObjectContainer)
+export default connect(mapStateToProps)(ObjectContainer)

browser/app/js/objects/ObjectItem.js

@@ -54,7 +54,9 @@ export const ObjectItem = ({
           href="#"
           onClick={e => {
             e.preventDefault()
-            onClick()
+            if (onClick) {
+              onClick()
+            }
           }}
         >
           {name}

browser/app/js/objects/ObjectsBulkActions.js

@@ -28,6 +28,15 @@ export class ObjectsBulkActions extends React.Component {
       showDeleteConfirmation: false
     }
   }
+  handleDownload() {
+    const { checkedObjects, clearChecked, downloadChecked, downloadObject } = this.props
+    if (checkedObjects.length === 1 && !checkedObjects[0].endsWith('/')) {
+      downloadObject(checkedObjects[0])
+      clearChecked()
+    } else {
+      downloadChecked()
+    }
+  }
   deleteChecked() {
     const { deleteChecked } = this.props
     deleteChecked()
@@ -39,24 +48,27 @@ export class ObjectsBulkActions extends React.Component {
     })
   }
   render() {
-    const { checkedObjectsCount, downloadChecked, clearChecked } = this.props
+    const { checkedObjects, clearChecked } = this.props
     return (
       <div
         className={
           "list-actions" +
           classNames({
-            " list-actions-toggled": checkedObjectsCount > 0
+            " list-actions-toggled": checkedObjects.length > 0
           })
         }
       >
         <span className="la-label">
-          <i className="fa fa-check-circle" /> {checkedObjectsCount} Objects
+          <i className="fa fa-check-circle" /> {checkedObjects.length}
+          {checkedObjects.length === 1 ? " Object " : " Objects "}
           selected
         </span>
         <span className="la-actions pull-right">
-          <button id="download-checked" onClick={downloadChecked}>
+          <button id="download-checked" onClick={this.handleDownload.bind(this)}>
             {" "}
-            Download all as zip{" "}
+            Download
+            {(checkedObjects.length === 1 && !checkedObjects[0].endsWith('/')) ? 
+            " object" : " all as zip" }{" "}
           </button>
         </span>
         <span className="la-actions pull-right">
@@ -86,13 +98,16 @@ export class ObjectsBulkActions extends React.Component {
 
 const mapStateToProps = state => {
   return {
-    checkedObjectsCount: getCheckedList(state).length
+    checkedObjects: getCheckedList(state)
   }
 }
 
 const mapDispatchToProps = dispatch => {
   return {
+    downloadObject: object => dispatch(actions.downloadObject(object)),
     downloadChecked: () => dispatch(actions.downloadCheckedObjects()),
+    downloadObject: object => dispatch(actions.downloadObject(object)),
+    resetCheckedList: () => dispatch(actions.resetCheckedList()),
     clearChecked: () => dispatch(actions.resetCheckedList()),
     deleteChecked: () => dispatch(actions.deleteCheckedObjects())
   }

browser/app/js/objects/__tests__/ObjectActions.test.js

@@ -88,8 +88,21 @@ describe("ObjectActions", () => {
         object={{ name: "obj1" }}
         currentPrefix={"pre1/"}
         showShareObjectModal={true}
+        shareObjectName={"obj1"}
       />
     )
     expect(wrapper.find("Connect(ShareObjectModal)").length).toBe(1)
   })
+
+  it("shouldn't render ShareObjectModal when the names of the objects don't match", () => {
+    const wrapper = shallow(
+      <ObjectActions
+        object={{ name: "obj1" }}
+        currentPrefix={"pre1/"}
+        showShareObjectModal={true}
+        shareObjectName={"obj2"}
+      />
+    )
+    expect(wrapper.find("Connect(ShareObjectModal)").length).toBe(0)
+  })
 })

browser/app/js/objects/__tests__/ObjectItem.test.js

@@ -28,9 +28,16 @@ describe("ObjectItem", () => {
     expect(wrapper.prop("data-type")).toBe("image")
   })
 
-  it("should call onClick when the object isclicked", () => {
+  it("shouldn't call onClick when the object isclicked", () => {
     const onClick = jest.fn()
-    const wrapper = shallow(<ObjectItem name={"test"} onClick={onClick} />)
+    const wrapper = shallow(<ObjectItem name={"test"} />)
+    wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
+    expect(onClick).not.toHaveBeenCalled()
+  })
+
+  it("should call onClick when the folder isclicked", () => {
+    const onClick = jest.fn()
+    const wrapper = shallow(<ObjectItem name={"test/"} onClick={onClick} />)
     wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
     expect(onClick).toHaveBeenCalled()
   })

browser/app/js/objects/__tests__/ObjectsBulkActions.test.js

@@ -20,19 +20,45 @@ import { ObjectsBulkActions } from "../ObjectsBulkActions"
 
 describe("ObjectsBulkActions", () => {
   it("should render without crashing", () => {
-    shallow(<ObjectsBulkActions checkedObjectsCount={0} />)
+    shallow(<ObjectsBulkActions checkedObjects={[]} />)
   })
 
   it("should show actions when checkObjectsCount is more than 0", () => {
-    const wrapper = shallow(<ObjectsBulkActions checkedObjectsCount={1} />)
+    const wrapper = shallow(<ObjectsBulkActions checkedObjects={["test"]} />)
     expect(wrapper.hasClass("list-actions-toggled")).toBeTruthy()
   })
 
-  it("should call downloadChecked when download button is clicked", () => {
+  it("should call downloadObject when single object is selected and download button is clicked", () => {
+    const downloadObject = jest.fn()
+    const clearChecked = jest.fn()
+    const wrapper = shallow(
+      <ObjectsBulkActions
+        checkedObjects={["test"]}
+        downloadObject={downloadObject}
+        clearChecked={clearChecked}
+      />
+    )
+    wrapper.find("#download-checked").simulate("click")
+    expect(downloadObject).toHaveBeenCalled()
+  })
+
+  it("should call downloadChecked when a folder is selected and download button is clicked", () => {
     const downloadChecked = jest.fn()
     const wrapper = shallow(
       <ObjectsBulkActions
-        checkedObjectsCount={1}
+        checkedObjects={["test/"]}
+        downloadChecked={downloadChecked}
+      />
+    )
+    wrapper.find("#download-checked").simulate("click")
+    expect(downloadChecked).toHaveBeenCalled()
+  })
+
+  it("should call downloadChecked when multiple objects are selected and download button is clicked", () => {
+    const downloadChecked = jest.fn()
+    const wrapper = shallow(
+      <ObjectsBulkActions
+        checkedObjects={["test1", "test2"]}
         downloadChecked={downloadChecked}
       />
     )
@@ -43,14 +69,14 @@ describe("ObjectsBulkActions", () => {
   it("should call clearChecked when close button is clicked", () => {
     const clearChecked = jest.fn()
     const wrapper = shallow(
-      <ObjectsBulkActions checkedObjectsCount={1} clearChecked={clearChecked} />
+      <ObjectsBulkActions checkedObjects={["test"]} clearChecked={clearChecked} />
     )
     wrapper.find("#close-bulk-actions").simulate("click")
     expect(clearChecked).toHaveBeenCalled()
   })
 
   it("shoud show DeleteObjectConfirmModal when delete-checked button is clicked", () => {
-    const wrapper = shallow(<ObjectsBulkActions checkedObjectsCount={1} />)
+    const wrapper = shallow(<ObjectsBulkActions checkedObjects={["test"]} />)
     wrapper.find("#delete-checked").simulate("click")
     wrapper.update()
     expect(wrapper.find("DeleteObjectConfirmModal").length).toBe(1)
@@ -60,7 +86,7 @@ describe("ObjectsBulkActions", () => {
     const deleteChecked = jest.fn()
     const wrapper = shallow(
       <ObjectsBulkActions
-        checkedObjectsCount={1}
+        checkedObjects={["test"]}
         deleteChecked={deleteChecked}
       />
     )

browser/app/js/objects/__tests__/actions.test.js

@@ -19,16 +19,29 @@ import thunk from "redux-thunk"
 import * as actionsObjects from "../actions"
 import * as alertActions from "../../alert/actions"
 import { minioBrowserPrefix } from "../../constants"
+import history from "../../history"
 
 jest.mock("../../web", () => ({
-  LoggedIn: jest.fn(() => true).mockReturnValueOnce(false),
-  ListObjects: jest.fn(() => {
-    return Promise.resolve({
-      objects: [{ name: "test1" }, { name: "test2" }],
-      istruncated: false,
-      nextmarker: "test2",
-      writable: false
-    })
+  LoggedIn: jest
+    .fn(() => true)
+    .mockReturnValueOnce(true)
+    .mockReturnValueOnce(false)
+    .mockReturnValueOnce(true)
+    .mockReturnValueOnce(true)
+    .mockReturnValueOnce(false),
+  ListObjects: jest.fn(({ bucketName }) => {
+    if (bucketName === "test-deny") {
+      return Promise.reject({
+        message: "listobjects is denied"
+      })
+    } else {
+      return Promise.resolve({
+        objects: [{ name: "test1" }, { name: "test2" }],
+        istruncated: false,
+        nextmarker: "test2",
+        writable: false
+      })
+    }
   }),
   RemoveObject: jest.fn(({ bucketName, objects }) => {
     if (!bucketName) {
@@ -160,6 +173,41 @@ describe("Objects actions", () => {
     })
   })
 
+  it("creates objects/RESET_LIST after failing to fetch the objects from bucket with ListObjects denied for LoggedIn users", () => {
+    const store = mockStore({
+      buckets: { currentBucket: "test-deny" },
+      objects: { currentPrefix: "" }
+    })
+    const expectedActions = [
+      {
+        type: "alert/SET",
+        alert: {
+          type: "danger",
+          message: "listobjects is denied",
+          id: alertActions.alertId,
+          autoClear: true
+        }
+      },
+      {
+        type: "object/RESET_LIST"
+      }
+    ]
+    return store.dispatch(actionsObjects.fetchObjects()).then(() => {
+      const actions = store.getActions()
+      expect(actions).toEqual(expectedActions)
+    })
+  })
+
+  it("redirect to login after failing to fetch the objects from bucket for non-LoggedIn users", () => {
+    const store = mockStore({
+      buckets: { currentBucket: "test-deny" },
+      objects: { currentPrefix: "" }
+    })
+    return store.dispatch(actionsObjects.fetchObjects()).then(() => {
+      expect(history.location.pathname.endsWith("/login")).toBeTruthy()
+    })
+  })
+
   it("creates objects/SET_SORT_BY and objects/SET_SORT_ORDER when sortObjects is called", () => {
     const store = mockStore({
       objects: {
@@ -244,7 +292,11 @@ describe("Objects actions", () => {
     const expectedActions = [
       {
         type: "alert/SET",
-        alert: { type: "danger", message: "Invalid bucket", id: 0 }
+        alert: {
+          type: "danger",
+          message: "Invalid bucket",
+          id: alertActions.alertId
+        }
       }
     ]
     return store.dispatch(actionsObjects.deleteObject("obj1")).then(() => {
@@ -355,7 +407,7 @@ describe("Objects actions", () => {
       store.dispatch(actionsObjects.downloadObject("obj1"))
       const url = `${
         window.location.origin
-      }${minioBrowserPrefix}/download/bk1/${encodeURI("pre1/obj1")}?token=''`
+      }${minioBrowserPrefix}/download/bk1/${encodeURI("pre1/obj1")}?token=`
       expect(setLocation).toHaveBeenCalledWith(url)
     })
 

browser/app/js/objects/actions.js

@@ -16,17 +16,15 @@
 
 import web from "../web"
 import history from "../history"
-import {
-  sortObjectsByName,
-  sortObjectsBySize,
-  sortObjectsByDate
-} from "../utils"
+import { sortObjectsByName, sortObjectsBySize, sortObjectsByDate } from "../utils"
 import { getCurrentBucket } from "../buckets/selectors"
 import { getCurrentPrefix, getCheckedList } from "./selectors"
 import * as alertActions from "../alert/actions"
+import * as bucketActions from "../buckets/actions"
 import { minioBrowserPrefix } from "../constants"
 
 export const SET_LIST = "objects/SET_LIST"
+export const RESET_LIST = "object/RESET_LIST"
 export const APPEND_LIST = "objects/APPEND_LIST"
 export const REMOVE = "objects/REMOVE"
 export const SET_SORT_BY = "objects/SET_SORT_BY"
@@ -45,6 +43,10 @@ export const setList = (objects, marker, isTruncated) => ({
   isTruncated
 })
 
+export const resetList = () => ({
+  type: RESET_LIST
+})
+
 export const appendList = (objects, marker, isTruncated) => ({
   type: APPEND_LIST,
   objects,
@@ -54,47 +56,54 @@ export const appendList = (objects, marker, isTruncated) => ({
 
 export const fetchObjects = append => {
   return function(dispatch, getState) {
-    const {
-      buckets: { currentBucket },
-      objects: { currentPrefix, marker }
-    } = getState()
+    const {buckets: {currentBucket}, objects: {currentPrefix, marker}} = getState()
     if (currentBucket) {
       return web
-      .ListObjects({
-        bucketName: currentBucket,
-        prefix: currentPrefix,
-        marker: append ? marker : ""
-      })
-      .then(res => {
-        let objects = []
-        if (res.objects) {
-          objects = res.objects.map(object => {
-            return {
-              ...object,
-              name: object.name.replace(currentPrefix, "")
-            }
-          })
-        }
-        if (append) {
-          dispatch(appendList(objects, res.nextmarker, res.istruncated))
-        } else {
-          dispatch(setList(objects, res.nextmarker, res.istruncated))
-          dispatch(setSortBy(""))
-          dispatch(setSortOrder(false))
-        }
-        dispatch(setPrefixWritable(res.writable))
-      })
-      .catch(err => {
-        dispatch(alertActions.set({ type: "danger", message: err.message }))
-        history.push("/login")
-      })
-    } 
+        .ListObjects({
+          bucketName: currentBucket,
+          prefix: currentPrefix,
+          marker: append ? marker : ""
+        })
+        .then(res => {
+          let objects = []
+          if (res.objects) {
+            objects = res.objects.map(object => {
+              return {
+                ...object,
+                name: object.name.replace(currentPrefix, "")
+              }
+            })
+          }
+          if (append) {
+            dispatch(appendList(objects, res.nextmarker, res.istruncated))
+          } else {
+            dispatch(setList(objects, res.nextmarker, res.istruncated))
+            dispatch(setSortBy(""))
+            dispatch(setSortOrder(false))
+          }
+          dispatch(setPrefixWritable(res.writable))
+        })
+        .catch(err => {
+          if (web.LoggedIn()) {
+            dispatch(
+              alertActions.set({
+                type: "danger",
+                message: err.message,
+                autoClear: true
+              })
+            )
+            dispatch(resetList())
+          } else {
+            history.push("/login")
+          }
+        })
+    }
   }
 }
 
 export const sortObjects = sortBy => {
   return function(dispatch, getState) {
-    const { objects } = getState()
+    const {objects} = getState()
     const sortOrder = objects.sortBy == sortBy ? !objects.sortOrder : true
     dispatch(setSortBy(sortBy))
     dispatch(setSortOrder(sortOrder))
@@ -194,30 +203,39 @@ export const shareObject = (object, days, hours, minutes) => {
     const currentPrefix = getCurrentPrefix(getState())
     const objectName = `${currentPrefix}${object}`
     const expiry = days * 24 * 60 * 60 + hours * 60 * 60 + minutes * 60
-    return web
-      .PresignedGet({
-        host: location.host,
-        bucket: currentBucket,
-        object: objectName,
-        expiry
-      })
-      .then(obj => {
-        dispatch(showShareObject(object, obj.url))
-        dispatch(
-          alertActions.set({
-            type: "success",
-            message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`
-          })
-        )
-      })
-      .catch(err => {
-        dispatch(
-          alertActions.set({
-            type: "danger",
-            message: err.message
-          })
-        )
-      })
+    if (web.LoggedIn()) {
+      return web
+        .PresignedGet({
+          host: location.host,
+          bucket: currentBucket,
+          object: objectName
+        })
+        .then(obj => {
+          dispatch(showShareObject(object, obj.url))
+          dispatch(
+            alertActions.set({
+              type: "success",
+              message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`
+            })
+          )
+        })
+        .catch(err => {
+          dispatch(
+            alertActions.set({
+              type: "danger",
+              message: err.message
+            })
+          )
+        })
+    } else {
+      dispatch(showShareObject(object, `${location.host}` + '/' + `${currentBucket}` + '/' + encodeURI(objectName)))
+      dispatch(
+        alertActions.set({
+          type: "success",
+          message: `Object shared.`
+        })
+      )
+    }
   }
 }
 
@@ -263,7 +281,7 @@ export const downloadObject = object => {
     } else {
       const url = `${
         window.location.origin
-      }${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=''`
+      }${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=`
       window.location = url
     }
   }
@@ -292,7 +310,7 @@ export const downloadCheckedObjects = () => {
       objects: getCheckedList(state)
     }
     if (!web.LoggedIn()) {
-      const requestUrl = location.origin + "/minio/zip?token=''"
+      const requestUrl = location.origin + "/minio/zip?token="
       downloadZip(requestUrl, req, dispatch)
     } else {
       return web
@@ -303,14 +321,13 @@ export const downloadCheckedObjects = () => {
           }${minioBrowserPrefix}/zip?token=${res.token}`
           downloadZip(requestUrl, req, dispatch)
         })
-        .catch(err =>
-          dispatch(
-            alertActions.set({
-              type: "danger",
-              message: err.message
-            })
-          )
+        .catch(err => dispatch(
+          alertActions.set({
+            type: "danger",
+            message: err.message
+          })
         )
+      )
     }
   }
 }
@@ -333,8 +350,7 @@ const downloadZip = (url, req, dispatch) => {
       var separator = req.prefix.length > 1 ? "-" : ""
 
       anchor.href = blobUrl
-      anchor.download =
-        req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
+      anchor.download = req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
 
       anchor.click()
       window.URL.revokeObjectURL(blobUrl)

browser/app/js/objects/reducer.js

@@ -50,6 +50,13 @@ export default (
         marker: action.marker,
         isTruncated: action.isTruncated
       }
+    case actionsObjects.RESET_LIST:
+      return {
+        ...state,
+        list: [],
+        marker: "",
+        isTruncated: false
+      }
     case actionsObjects.APPEND_LIST:
       return {
         ...state,

browser/app/less/inc/header.less

@@ -44,9 +44,9 @@
 
 
 /*--------------------------
-    Disk usage
+    Disk used
 ----------------------------*/
-.feh-usage {
+.feh-used {
     margin-top: 12px;
     max-width: 285px;
 

browser/app/less/inc/list.less

@@ -100,10 +100,24 @@ div.fesl-row {
         }
     }
 
+    .fesl-item-name {
+        a {
+            cursor: default;
+        }
+    }
+
     /*--------------------------
         Icons
     ----------------------------*/
-    &[data-type=folder] { .list-type(#a1d6dd, '\f114'); }
+    &[data-type=folder] {
+        .list-type(#a1d6dd, '\f114');
+
+        .fesl-item-name {
+            a {
+                cursor: pointer;
+            }
+        }
+    }
     &[data-type=pdf] {.list-type(#fa7775, '\f1c1'); }
     &[data-type=zip] { .list-type(#427089, '\f1c6'); }
     &[data-type=audio] { .list-type(#009688, '\f1c7'); }

browser/build.js

@@ -70,9 +70,9 @@ async.waterfall([
       commitId = stdout.replace('\n', '')
       if (commitId.length !== 40) throw new Error('commitId invalid : ' + commitId)
       assetsFileName = 'ui-assets.go';
-      var cmd = 'go-bindata-assetfs -pkg browser -nocompress=true production/...'
+      var cmd = 'go-bindata-assetfs -o bindata_assetfs.go -pkg browser -nocompress=true production/...'
       if (!isProduction) {
-        cmd = 'go-bindata-assetfs -pkg browser -nocompress=true dev/...'
+        cmd = 'go-bindata-assetfs -o bindata_assetfs.go -pkg browser -nocompress=true dev/...'
       }
       console.log('Running', cmd)
       exec(cmd, cb)

browser/package.json

@@ -13,8 +13,7 @@
     "setupTestFrameworkScriptFile": "./app/js/jest/setup.js",
     "testURL": "https://localhost:8080",
     "moduleNameMapper": {
-      "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$":
-        "<rootDir>/app/js/jest/__mocks__/fileMock.js",
+      "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/app/js/jest/__mocks__/fileMock.js",
       "\\.(css|scss)$": "identity-obj-proxy"
     }
   },
@@ -40,29 +39,31 @@
     "babel-preset-es2015": "^6.14.0",
     "babel-preset-react": "^6.11.1",
     "babel-register": "^6.14.0",
-    "copy-webpack-plugin": "^0.3.3",
+    "copy-webpack-plugin": "^4.6.0",
     "css-loader": "^0.23.1",
     "enzyme": "^3.3.0",
     "enzyme-adapter-react-16": "^1.1.1",
     "esformatter": "^0.10.0",
     "esformatter-jsx": "^7.4.1",
     "esformatter-jsx-ignore": "^1.0.6",
-    "html-webpack-plugin": "^2.30.1",
+    "html-webpack-plugin": "^3.2.0",
     "jest": "^22.1.4",
     "jest-enzyme": "^4.0.2",
     "json-loader": "^0.5.4",
-    "less": "^2.7.1",
-    "less-loader": "^2.2.3",
-    "purifycss-webpack-plugin": "^2.0.3",
+    "less": "^3.9.0",
+    "less-loader": "^4.1.0",
+    "purgecss-webpack-plugin": "^1.4.0",
     "style-loader": "^0.13.1",
     "url-loader": "^0.5.7",
-    "webpack-dev-server": "^2.11.1"
+    "webpack-cli": "^3.2.0",
+    "webpack-dev-server": "^3.1.14"
   },
   "dependencies": {
     "bootstrap": "^3.3.6",
     "classnames": "^2.2.3",
     "expect": "^1.20.2",
     "font-awesome": "^4.7.0",
+    "glob-all": "^3.1.0",
     "history": "^4.7.2",
     "humanize": "0.0.9",
     "identity-obj-proxy": "^3.0.0",
@@ -89,6 +90,6 @@
     "reselect": "^3.0.1",
     "superagent": "^3.8.2",
     "superagent-es6-promise": "^1.0.0",
-    "webpack": "^3.10.0"
+    "webpack": "^4.28.3"
   }
 }

browser/webpack.config.js

@@ -16,11 +16,13 @@
 
 var webpack = require('webpack')
 var path = require('path')
+var glob = require('glob-all')
 var CopyWebpackPlugin = require('copy-webpack-plugin')
-var purify = require("purifycss-webpack-plugin")
+var PurgecssPlugin = require('purgecss-webpack-plugin')
 
 var exports = {
   context: __dirname,
+  mode: 'development',
   entry: [
     path.resolve(__dirname, 'app/index.js')
   ],
@@ -99,12 +101,11 @@ var exports = {
       {from: 'app/index.html'}
     ]),
     new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),
-    new purify({
-        basePath: __dirname,
-        paths: [
-            "app/index.html",
-            "app/js/*.js"
-        ]
+    new PurgecssPlugin({
+      paths: glob.sync([
+        path.join(__dirname, 'app/index.html'),
+        path.join(__dirname, 'app/js/*.js')
+      ])
     })
   ]
 }

browser/webpack.production.config.js

@@ -16,11 +16,13 @@
 
 var webpack = require('webpack')
 var path = require('path')
+var glob = require('glob-all')
 var CopyWebpackPlugin = require('copy-webpack-plugin')
-var purify = require("purifycss-webpack-plugin")
+var PurgecssPlugin = require('purgecss-webpack-plugin')
 
 var exports = {
   context: __dirname,
+  mode: 'production',
   entry: [
     path.resolve(__dirname, 'app/index.js')
   ],
@@ -74,16 +76,12 @@ var exports = {
       {from: 'app/img/logo.svg'},
       {from: 'app/index.html'}
     ]),
-    new webpack.DefinePlugin({
-        'process.env.NODE_ENV': '"production"'
-    }),
     new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),
-    new purify({
-        basePath: __dirname,
-        paths: [
-            "app/index.html",
-            "app/js/*.js"
-        ]
+    new PurgecssPlugin({
+      paths: glob.sync([
+        path.join(__dirname, 'app/index.html'),
+        path.join(__dirname, 'app/js/*.js')
+      ])
     })
   ]
 }

buildscripts/checkdeps.sh

@@ -34,7 +34,7 @@ _init() {
 
 ## FIXME:
 ## In OSX, 'readlink -f' option does not exist, hence
-## we have our own readlink -f behaviour here.
+## we have our own readlink -f behavior here.
 ## Once OSX has the option, below function is good enough.
 ## 
 ## readlink() {
@@ -89,11 +89,11 @@ check_minimum_version() {
 
 assert_is_supported_arch() {
     case "${ARCH}" in
-        x86_64 | amd64 | aarch64 | arm* )
+        x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x )
             return
             ;;
         *)
-            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, arm*]"
+            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x]"
             exit 1
     esac
 }

buildscripts/cross-compile.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Enable tracing if set.
+[ -n "$BASH_XTRACEFD" ] && set -ex
+
+function _init() {
+    ## All binaries are static make sure to disable CGO.
+    export CGO_ENABLED=0
+
+    ## List of architectures and OS to test coss compilation.
+    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64"
+}
+
+function _build_and_sign() {
+    local osarch=$1
+    IFS=/ read -r -a arr <<<"$osarch"
+    os="${arr[0]}"
+    arch="${arr[1]}"
+    package=$(go list -f '{{.ImportPath}}')
+    printf -- "--> %15s:%s\n" "${osarch}" "${package}"
+
+    # Go build to build the binary.
+    export GOOS=$os
+    export GOARCH=$arch
+    go build -tags kqueue -o /dev/null
+}
+
+function main() {
+    echo "Testing builds for OS/Arch: ${SUPPORTED_OSARCH}"
+    for each_osarch in ${SUPPORTED_OSARCH}; do
+        _build_and_sign "${each_osarch}"
+    done
+}
+
+_init && main "$@"

buildscripts/gateway-tests.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Minio Cloud Storage, (C) 2019 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+set -E
+set -o pipefail
+
+export SERVER_ENDPOINT=127.0.0.1:24240
+export ENABLE_HTTPS=0
+export ACCESS_KEY=minio
+export SECRET_KEY=minio123
+export MINIO_ACCESS_KEY=minio
+export MINIO_SECRET_KEY=minio123
+export AWS_ACCESS_KEY_ID=minio
+export AWS_SECRET_ACCESS_KEY=minio123
+
+trap "cat server.log;cat gateway.log" SIGHUP SIGINT SIGTERM
+
+./minio --quiet --json server data --address 127.0.0.1:24242 > server.log &
+sleep 3
+./minio --quiet --json gateway s3 http://127.0.0.1:24242 --address 127.0.0.1:24240 > gateway.log &
+sleep 3
+
+mkdir -p /mint
+git clone https://github.com/minio/mint /mint
+cd /mint
+
+export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint}
+export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core"
+export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security"
+export MINT_MODE="full"
+export WGET="wget --quiet --no-check-certificate"
+
+go get github.com/go-ini/ini
+
+./create-data-files.sh
+./preinstall.sh
+
+# install mint app packages
+for pkg in "build"/*/install.sh; do
+    echo "Running $pkg"
+    $pkg
+done
+
+./postinstall.sh
+
+/mint/entrypoint.sh || cat server.log gateway.log fail

buildscripts/gen-ldflags.go

@@ -33,6 +33,7 @@ func genLDFlags(version string) string {
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
 	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
+	ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT")
 	return ldflagsStr
 }
 

buildscripts/go-coverage.sh

@@ -4,7 +4,7 @@ set -e
 echo "" > coverage.txt
 
 for d in $(go list ./... | grep -v browser); do
-    go test -coverprofile=profile.out -covermode=atomic "$d"
+    CGO_ENABLED=0 go test -v -coverprofile=profile.out -covermode=atomic "$d"
     if [ -f profile.out ]; then
         cat profile.out >> coverage.txt
         rm profile.out

buildscripts/verify-build.sh

@@ -68,9 +68,41 @@ function start_minio_erasure_sets()
     echo "$minio_pid"
 }
 
+function start_minio_dist_erasure_sets_ipv6()
+{
+    declare -a minio_pids
+    export MINIO_ACCESS_KEY=$ACCESS_KEY
+    export MINIO_SECRET_KEY=$SECRET_KEY
+    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9000.log" 2>&1 &
+    minio_pids[0]=$!
+    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9001.log" 2>&1 &
+    minio_pids[1]=$!
+    "${MINIO[@]}" server --address="[::1]:9002" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9002.log" 2>&1 &
+    minio_pids[2]=$!
+    "${MINIO[@]}" server --address="[::1]:9003" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9003.log" 2>&1 &
+    minio_pids[3]=$!
+    "${MINIO[@]}" server --address="[::1]:9004" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9004.log" 2>&1 &
+    minio_pids[4]=$!
+    "${MINIO[@]}" server --address="[::1]:9005" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9005.log" 2>&1 &
+    minio_pids[5]=$!
+    "${MINIO[@]}" server --address="[::1]:9006" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9006.log" 2>&1 &
+    minio_pids[6]=$!
+    "${MINIO[@]}" server --address="[::1]:9007" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9007.log" 2>&1 &
+    minio_pids[7]=$!
+    "${MINIO[@]}" server --address="[::1]:9008" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9008.log" 2>&1 &
+    minio_pids[8]=$!
+    "${MINIO[@]}" server --address="[::1]:9009" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9009.log" 2>&1 &
+    minio_pids[9]=$!
+
+    sleep 35
+    echo "${minio_pids[@]}"
+}
+
 function start_minio_dist_erasure_sets()
 {
     declare -a minio_pids
+    export MINIO_ACCESS_KEY=$ACCESS_KEY
+    export MINIO_SECRET_KEY=$SECRET_KEY
     "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
     minio_pids[0]=$!
     "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
@@ -99,6 +131,8 @@ function start_minio_dist_erasure_sets()
 function start_minio_dist_erasure()
 {
     declare -a minio_pids
+    export MINIO_ACCESS_KEY=$ACCESS_KEY
+    export MINIO_SECRET_KEY=$SECRET_KEY
     "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
     minio_pids[0]=$!
     "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
@@ -157,6 +191,34 @@ function run_test_erasure_sets() {
     return "$rv"
 }
 
+function run_test_dist_erasure_sets_ipv6()
+{
+    minio_pids=( $(start_minio_dist_erasure_sets_ipv6) )
+
+    export SERVER_ENDPOINT="[::1]:9000"
+
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    for pid in "${minio_pids[@]}"; do
+        kill "$pid"
+    done
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        for i in $(seq 0 9); do
+            echo "server$i log:"
+            cat "$WORK_DIR/dist-minio-v6-900$i.log"
+        done
+    fi
+
+    for i in $(seq 0 9); do
+        rm -f "$WORK_DIR/dist-minio-v6-900$i.log"
+    done
+
+    return "$rv"
+}
+
 function run_test_dist_erasure_sets()
 {
     minio_pids=( $(start_minio_dist_erasure_sets) )
@@ -233,6 +295,7 @@ function run_test_gateway_s3()
 {
     minio_pid="$(start_minio_gateway_s3)"
 
+    export SERVER_ENDPOINT="127.0.0.1:9000"
     export ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
     export SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
@@ -275,6 +338,7 @@ function __init__()
         exit 1
     fi
 
+    sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS"
     chmod a+x "$FUNCTIONAL_TESTS"
 }
 
@@ -315,6 +379,13 @@ function main()
         exit 1
     fi
 
+    echo "Testing in Distributed Erasure setup as sets with ipv6"
+    if ! run_test_dist_erasure_sets_ipv6; then
+        echo "FAILED"
+        rm -fr "$WORK_DIR"
+        exit 1
+    fi
+
     echo "Testing in Gateway S3 setup"
     if ! run_test_gateway_s3; then
         echo "FAILED"

cmd/acl-handlers.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -54,40 +55,44 @@ type accessControlPolicy struct {
 // This operation uses the ACL
 // subresource to return the ACL of a specified bucket.
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketACL")
+	ctx := newContext(r, w, "GetBucketACL")
+
+	defer logger.AuditLog(w, r, "GetBucketACL", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allow getBucketACL if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	acl := &accessControlPolicy{}
 	acl.AccessControlList.Grants = append(acl.AccessControlList.Grants, grant{
 		Grantee: grantee{
-			Type: "CanonicalUser",
+			XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
+			XMLXSI: "CanonicalUser",
+			Type:   "CanonicalUser",
 		},
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -99,7 +104,9 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 // This operation uses the ACL
 // subresource to return the ACL of a specified object.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetObjectACL")
+	ctx := newContext(r, w, "GetObjectACL")
+
+	defer logger.AuditLog(w, r, "GetObjectACL", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -107,33 +114,35 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allow getObjectACL if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Before proceeding validate if object exists.
-	_, err := objAPI.GetObjectInfo(ctx, bucket, object)
+	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	acl := &accessControlPolicy{}
 	acl.AccessControlList.Grants = append(acl.AccessControlList.Grants, grant{
 		Grantee: grantee{
-			Type: "CanonicalUser",
+			XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
+			XMLXSI: "CanonicalUser",
+			Type:   "CanonicalUser",
 		},
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/admin-heal-ops.go

@@ -20,12 +20,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 // healStatusSummary - overall short summary of a healing sequence
@@ -57,11 +60,10 @@ const (
 var (
 	errHealIdleTimeout      = fmt.Errorf("healing results were not consumed for too long")
 	errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
-	errHealStopSignalled    = fmt.Errorf("heal stop signalled")
+	errHealStopSignalled    = fmt.Errorf("heal stop signaled")
 
-	errFnHealFromAPIErr = func(err error) error {
-		errCode := toAPIErrorCode(err)
-		apiErr := getAPIError(errCode)
+	errFnHealFromAPIErr = func(ctx context.Context, err error) error {
+		apiErr := toAPIError(ctx, err)
 		return fmt.Errorf("Heal internal error: %s: %s",
 			apiErr.Code, apiErr.Description)
 	}
@@ -110,6 +112,32 @@ func initAllHealState(isErasureMode bool) {
 	globalAllHealState = allHealState{
 		healSeqMap: make(map[string]*healSequence),
 	}
+
+	go globalAllHealState.periodicHealSeqsClean()
+}
+
+func (ahs *allHealState) periodicHealSeqsClean() {
+	// Launch clean-up routine to remove this heal sequence (after
+	// it ends) from the global state after timeout has elapsed.
+	ticker := time.NewTicker(time.Minute * 5)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			now := UTCNow()
+			ahs.Lock()
+			for path, h := range ahs.healSeqMap {
+				if h.hasEnded() && h.endTime.Add(keepHealSeqStateDuration).Before(now) {
+					delete(ahs.healSeqMap, path)
+				}
+			}
+			ahs.Unlock()
+		case <-GlobalServiceDoneCh:
+			// server could be restarting - need
+			// to exit immediately
+			return
+		}
+	}
 }
 
 // getHealSequence - Retrieve a heal sequence by path. The second
@@ -121,6 +149,35 @@ func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists b
 	return h, exists
 }
 
+func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
+	var hsp madmin.HealStopSuccess
+	he, exists := ahs.getHealSequence(path)
+	if !exists {
+		hsp = madmin.HealStopSuccess{
+			ClientToken: "invalid",
+			StartTime:   UTCNow(),
+		}
+	} else {
+		hsp = madmin.HealStopSuccess{
+			ClientToken:   he.clientToken,
+			ClientAddress: he.clientAddress,
+			StartTime:     he.startTime,
+		}
+
+		he.stop()
+		for !he.hasEnded() {
+			time.Sleep(1 * time.Second)
+		}
+		ahs.Lock()
+		defer ahs.Unlock()
+		// Heal sequence explicitly stopped, remove it.
+		delete(ahs.healSeqMap, path)
+	}
+
+	b, err := json.Marshal(&hsp)
+	return b, toAdminAPIErr(context.Background(), err)
+}
+
 // LaunchNewHealSequence - launches a background routine that performs
 // healing according to the healSequence argument. For each heal
 // sequence, state is stored in the `globalAllHealState`, which is a
@@ -132,7 +189,7 @@ func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists b
 // background routine to clean up heal results after the
 // aforementioned duration.
 func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
-	respBytes []byte, errCode APIErrorCode, errMsg string) {
+	respBytes []byte, apiErr APIError, errMsg string) {
 
 	existsAndLive := false
 	he, exists := ahs.getHealSequence(h.path)
@@ -141,22 +198,21 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 			existsAndLive = true
 		}
 	}
+
 	if existsAndLive {
 		// A heal sequence exists on the given path.
 		if h.forceStarted {
-			// stop the running heal sequence - wait for
-			// it to finish.
+			// stop the running heal sequence - wait for it to finish.
 			he.stop()
 			for !he.hasEnded() {
-				time.Sleep(10 * time.Second)
+				time.Sleep(1 * time.Second)
 			}
 		} else {
 			errMsg = "Heal is already running on the given path " +
 				"(use force-start option to stop and start afresh). " +
-				fmt.Sprintf("The heal was started by IP %s at %s",
-					h.clientAddress, h.startTime)
-
-			return nil, ErrHealAlreadyRunning, errMsg
+				fmt.Sprintf("The heal was started by IP %s at %s, token is %s",
+					h.clientAddress, h.startTime.Format(http.TimeFormat), h.clientToken)
+			return nil, errorCodes.ToAPIErr(ErrHealAlreadyRunning), errMsg
 		}
 	}
 
@@ -171,7 +227,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 
 			errMsg = "The provided heal sequence path overlaps with an existing " +
 				fmt.Sprintf("heal path: %s", k)
-			return nil, ErrHealOverlappingPaths, errMsg
+			return nil, errorCodes.ToAPIErr(ErrHealOverlappingPaths), errMsg
 		}
 	}
 
@@ -181,51 +237,16 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	// Launch top-level background heal go-routine
 	go h.healSequenceStart()
 
-	// Launch clean-up routine to remove this heal sequence (after
-	// it ends) from the global state after timeout has elapsed.
-	go func() {
-		var keepStateTimeout <-chan time.Time
-		ticker := time.NewTicker(time.Minute)
-		defer ticker.Stop()
-		everyMinute := ticker.C
-		for {
-			select {
-			// Check every minute if heal sequence has ended.
-			case <-everyMinute:
-				if h.hasEnded() {
-					keepStateTimeout = time.After(keepHealSeqStateDuration)
-					everyMinute = nil
-				}
-
-			// This case does not fire until the heal
-			// sequence completes.
-			case <-keepStateTimeout:
-				// Heal sequence has ended, keep
-				// results state duration has elapsed,
-				// so purge state.
-				ahs.Lock()
-				defer ahs.Unlock()
-				delete(ahs.healSeqMap, h.path)
-				return
-
-			case <-globalServiceDoneCh:
-				// server could be restarting - need
-				// to exit immediately
-				return
-			}
-		}
-	}()
-
 	b, err := json.Marshal(madmin.HealStartSuccess{
 		ClientToken:   h.clientToken,
 		ClientAddress: h.clientAddress,
 		StartTime:     h.startTime,
 	})
 	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return nil, ErrInternalError, ""
+		logger.LogIf(h.ctx, err)
+		return nil, toAPIError(h.ctx, err), ""
 	}
-	return b, ErrNone, ""
+	return b, noError, ""
 }
 
 // PopHealStatusJSON - Called by heal-status API. It fetches the heal
@@ -270,7 +291,7 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 
 	jbytes, err := json.Marshal(h.currentStatus)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(h.ctx, err)
 		return nil, ErrInternalError
 	}
 
@@ -283,12 +304,15 @@ type healSequence struct {
 	// bucket, and prefix on which heal seq. was initiated
 	bucket, objPrefix string
 
-	// path is just bucket + "/" + objPrefix
+	// path is just pathJoin(bucket, objPrefix)
 	path string
 
 	// time at which heal sequence was started
 	startTime time.Time
 
+	// time at which heal sequence has ended
+	endTime time.Time
+
 	// Heal client info
 	clientToken, clientAddress string
 
@@ -301,7 +325,7 @@ type healSequence struct {
 	// current accumulated status of the heal sequence
 	currentStatus healSequenceStatus
 
-	// channel signalled by background routine when traversal has
+	// channel signaled by background routine when traversal has
 	// completed
 	traverseAndHealDoneCh chan error
 
@@ -328,7 +352,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 	return &healSequence{
 		bucket:        bucket,
 		objPrefix:     objPrefix,
-		path:          bucket + "/" + objPrefix,
+		path:          pathJoin(bucket, objPrefix),
 		startTime:     UTCNow(),
 		clientToken:   mustGetUUID(),
 		clientAddress: clientAddr,
@@ -383,7 +407,6 @@ func (h *healSequence) stop() {
 // sequence automatically resumes. The return value indicates if the
 // operation succeeded.
 func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
-
 	// start a timer to keep an upper time limit to find an empty
 	// slot to add the given heal result - if no slot is found it
 	// means that the server is holding the maximum amount of
@@ -441,7 +464,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 	h.currentStatus.updateLock.Unlock()
 
 	// This is a "safe" point for the heal sequence to quit if
-	// signalled externally.
+	// signaled externally.
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
@@ -467,6 +490,7 @@ func (h *healSequence) healSequenceStart() {
 
 	select {
 	case err, ok := <-h.traverseAndHealDoneCh:
+		h.endTime = UTCNow()
 		h.currentStatus.updateLock.Lock()
 		defer h.currentStatus.updateLock.Unlock()
 		// Heal traversal is complete.
@@ -480,6 +504,7 @@ func (h *healSequence) healSequenceStart() {
 		}
 
 	case <-h.stopSignalCh:
+		h.endTime = UTCNow()
 		h.currentStatus.updateLock.Lock()
 		h.currentStatus.Summary = healStoppedStatus
 		h.currentStatus.FailureDetail = errHealStopSignalled.Error()
@@ -519,6 +544,12 @@ func (h *healSequence) traverseAndHeal() {
 	// Start with format healing
 	checkErr(h.healDiskFormat)
 
+	// Start healing the config prefix.
+	checkErr(h.healMinioSysMeta(minioConfigPrefix))
+
+	// Start healing the bucket config prefix.
+	checkErr(h.healMinioSysMeta(bucketConfigPrefix))
+
 	// Heal buckets and objects
 	checkErr(h.healBuckets)
 
@@ -529,9 +560,74 @@ func (h *healSequence) traverseAndHeal() {
 	close(h.traverseAndHealDoneCh)
 }
 
+// healMinioSysMeta - heals all files under a given meta prefix, returns a function
+// which in-turn heals the respective meta directory path and any files in int.
+func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
+	return func() error {
+		// Get current object layer instance.
+		objectAPI := newObjectLayerFn()
+		if objectAPI == nil {
+			return errServerNotInitialized
+		}
+
+		// NOTE: Healing on meta is run regardless
+		// of any bucket being selected, this is to ensure that
+		// meta are always upto date and correct.
+		marker := ""
+		isTruncated := true
+		for isTruncated {
+			if globalHTTPServer != nil {
+				// Wait at max 1 minute for an inprogress request
+				// before proceeding to heal
+				waitCount := 60
+				// Any requests in progress, delay the heal.
+				for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+					waitCount--
+					time.Sleep(1 * time.Second)
+				}
+			}
+
+			// Lists all objects under `config` prefix.
+			objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, minioMetaBucket, metaPrefix,
+				marker, "", 1000)
+			if err != nil {
+				return errFnHealFromAPIErr(h.ctx, err)
+			}
+
+			for index := range objectInfos.Objects {
+				if h.isQuitting() {
+					return errHealStopSignalled
+				}
+				o := objectInfos.Objects[index]
+				res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun, h.settings.Remove)
+				// Object might have been deleted, by the time heal
+				// was attempted we ignore this file an move on.
+				if isErrObjectNotFound(herr) {
+					continue
+				}
+				if herr != nil {
+					return herr
+				}
+				res.Type = madmin.HealItemBucketMetadata
+				if err = h.pushHealResultItem(res); err != nil {
+					return err
+				}
+			}
+
+			isTruncated = objectInfos.IsTruncated
+			marker = objectInfos.NextMarker
+		}
+		return nil
+	}
+}
+
 // healDiskFormat - heals format.json, return value indicates if a
 // failure error occurred.
 func (h *healSequence) healDiskFormat() error {
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -542,13 +638,18 @@ func (h *healSequence) healDiskFormat() error {
 	// return any error, ignore error returned when disks have
 	// already healed.
 	if err != nil && err != errNoHealRequired {
-		return errFnHealFromAPIErr(err)
+		return errFnHealFromAPIErr(h.ctx, err)
 	}
 
 	// Healing succeeded notify the peers to reload format and re-initialize disks.
 	// We will not notify peers only if healing succeeded.
 	if err == nil {
-		peersReInitFormat(globalAdminPeers, h.settings.DryRun)
+		for _, nerr := range globalNotificationSys.ReloadFormat(h.settings.DryRun) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(h.ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(h.ctx, nerr.Err)
+			}
+		}
 	}
 
 	// Push format heal result
@@ -557,6 +658,10 @@ func (h *healSequence) healDiskFormat() error {
 
 // healBuckets - check for all buckets heal or just particular bucket.
 func (h *healSequence) healBuckets() error {
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
 		return h.healBucket(h.bucket)
@@ -570,7 +675,7 @@ func (h *healSequence) healBuckets() error {
 
 	buckets, err := objectAPI.ListBucketsHeal(h.ctx)
 	if err != nil {
-		return errFnHealFromAPIErr(err)
+		return errFnHealFromAPIErr(h.ctx, err)
 	}
 
 	for _, bucket := range buckets {
@@ -584,36 +689,29 @@ func (h *healSequence) healBuckets() error {
 
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(bucket string) error {
-	if h.isQuitting() {
-		return errHealStopSignalled
-	}
-
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
-	results, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun)
-	// push any available results before checking for error
-	for _, result := range results {
-		if perr := h.pushHealResultItem(result); perr != nil {
-			return perr
-		}
-	}
+	result, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun, h.settings.Remove)
 	// handle heal-bucket error
 	if err != nil {
 		return err
 	}
 
+	if err = h.pushHealResultItem(result); err != nil {
+		return err
+	}
+
 	if !h.settings.Recursive {
 		if h.objPrefix != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			_, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix)
+			_, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{})
 			if err == nil {
-				err = h.healObject(bucket, h.objPrefix)
-				if err != nil {
+				if err = h.healObject(bucket, h.objPrefix); err != nil {
 					return err
 				}
 			}
@@ -622,17 +720,40 @@ func (h *healSequence) healBucket(bucket string) error {
 		return nil
 	}
 
+	entries := runtime.NumCPU()
+
 	marker := ""
 	isTruncated := true
 	for isTruncated {
-		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, bucket,
-			h.objPrefix, marker, "", 1000)
-		if err != nil {
-			return errFnHealFromAPIErr(err)
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to heal
+			waitCount := 60
+			// Any requests in progress, delay the heal.
+			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
 		}
 
-		for _, o := range objectInfos.Objects {
-			if err := h.healObject(o.Bucket, o.Name); err != nil {
+		// Heal numCPU * nodes objects at a time.
+		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, bucket,
+			h.objPrefix, marker, "", entries)
+		if err != nil {
+			return errFnHealFromAPIErr(h.ctx, err)
+		}
+
+		g := errgroup.WithNErrs(len(objectInfos.Objects))
+		for index := range objectInfos.Objects {
+			index := index
+			g.Go(func() error {
+				o := objectInfos.Objects[index]
+				return h.healObject(o.Bucket, o.Name)
+			}, index)
+		}
+
+		for _, err := range g.Wait() {
+			if err != nil {
 				return err
 			}
 		}
@@ -655,7 +776,10 @@ func (h *healSequence) healObject(bucket, object string) error {
 		return errServerNotInitialized
 	}
 
-	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun)
+	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove)
+	if isErrObjectNotFound(err) {
+		return nil
+	}
 	if err != nil {
 		hri.Detail = err.Error()
 	}

cmd/admin-router.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,48 +31,93 @@ type adminAPIHandlers struct {
 }
 
 // registerAdminRouter - Add handler functions for each service REST API routes.
-func registerAdminRouter(router *mux.Router) {
+func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) {
 
 	adminAPI := adminAPIHandlers{}
 	// Admin router
 	adminRouter := router.PathPrefix(adminAPIPathPrefix).Subrouter()
 
 	// Version handler
-	adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(adminAPI.VersionHandler)
+	adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(httpTraceAll(adminAPI.VersionHandler))
 
 	adminV1Router := adminRouter.PathPrefix("/v1").Subrouter()
 
 	/// Service operations
 
 	// Service status
-	adminV1Router.Methods(http.MethodGet).Path("/service").HandlerFunc(adminAPI.ServiceStatusHandler)
+	adminV1Router.Methods(http.MethodGet).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceStatusHandler))
 
 	// Service restart and stop - TODO
-	adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(adminAPI.ServiceStopNRestartHandler)
+	adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceStopNRestartHandler))
 
 	// Info operations
-	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(adminAPI.ServerInfoHandler)
+	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
 
-	/// Lock operations
+	if globalIsDistXL || globalIsXL {
+		/// Heal operations
 
-	// List Locks
-	adminV1Router.Methods(http.MethodGet).Path("/locks").HandlerFunc(adminAPI.ListLocksHandler)
-	// Clear locks
-	adminV1Router.Methods(http.MethodDelete).Path("/locks").HandlerFunc(adminAPI.ClearLocksHandler)
+		// Heal processing endpoint.
+		adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
-	/// Heal operations
+		/// Health operations
 
-	// Heal processing endpoint.
-	adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(adminAPI.HealHandler)
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(adminAPI.HealHandler)
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(adminAPI.HealHandler)
+	}
+	// Performance command - return performance details based on input type
+	adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
+
+	// Profiling operations
+	adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
+		Queries("profilerType", "{profilerType:.*}")
+	adminV1Router.Methods(http.MethodGet).Path("/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
 
 	/// Config operations
+	if enableConfigOps {
+		// Update credentials
+		adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
+		// Get config
+		adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
+		// Set config
+		adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
 
-	// Update credentials
-	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(adminAPI.UpdateCredentialsHandler)
-	// Get config
-	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(adminAPI.GetConfigHandler)
-	// Set config
-	adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(adminAPI.SetConfigHandler)
+		// Get config keys/values
+		adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
+		// Set config keys/values
+		adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
+	}
+
+	if enableIAMOps {
+		// -- IAM APIs --
+
+		// Add policy IAM
+		adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name",
+			"{name:.*}")
+
+		// Add user IAM
+		adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
+		adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
+			Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
+		adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
+			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
+
+		// Remove policy IAM
+		adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+
+		// Remove user IAM
+		adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+
+		// List users
+		adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+
+		// List policies
+		adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+	}
+
+	// -- Top APIs --
+	// Top locks
+	adminV1Router.Methods(http.MethodGet).Path("/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+
+	// If none of the routes match, return error.
+	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
 }

cmd/admin-rpc-client.go

@@ -1,648 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014, 2015, 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net"
-	"os"
-	"path"
-	"path/filepath"
-	"sort"
-	"sync"
-	"time"
-
-	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/cmd/logger"
-)
-
-const (
-	// Admin service names
-	signalServiceRPC  = "Admin.SignalService"
-	reInitFormatRPC   = "Admin.ReInitFormat"
-	listLocksRPC      = "Admin.ListLocks"
-	serverInfoDataRPC = "Admin.ServerInfoData"
-	getConfigRPC      = "Admin.GetConfig"
-	writeTmpConfigRPC = "Admin.WriteTmpConfig"
-	commitConfigRPC   = "Admin.CommitConfig"
-)
-
-// localAdminClient - represents admin operation to be executed locally.
-type localAdminClient struct {
-}
-
-// remoteAdminClient - represents admin operation to be executed
-// remotely, via RPC.
-type remoteAdminClient struct {
-	*AuthRPCClient
-}
-
-// adminCmdRunner - abstracts local and remote execution of admin
-// commands like service stop and service restart.
-type adminCmdRunner interface {
-	SignalService(s serviceSignal) error
-	ReInitFormat(dryRun bool) error
-	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
-	ServerInfoData() (ServerInfoData, error)
-	GetConfig() ([]byte, error)
-	WriteTmpConfig(tmpFileName string, configBytes []byte) error
-	CommitConfig(tmpFileName string) error
-}
-
-var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
-
-// SignalService - sends a restart or stop signal to the local server
-func (lc localAdminClient) SignalService(s serviceSignal) error {
-	switch s {
-	case serviceRestart, serviceStop:
-		globalServiceSignalCh <- s
-	default:
-		return errUnsupportedSignal
-	}
-	return nil
-}
-
-// ReInitFormat - re-initialize disk format.
-func (lc localAdminClient) ReInitFormat(dryRun bool) error {
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	return objectAPI.ReloadFormat(context.Background(), dryRun)
-}
-
-// ListLocks - Fetches lock information from local lock instrumentation.
-func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	// check if objectLayer is initialized, if not return.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return nil, errServerNotInitialized
-	}
-	return objectAPI.ListLocks(context.Background(), bucket, prefix, duration)
-}
-
-func (rc remoteAdminClient) SignalService(s serviceSignal) (err error) {
-	switch s {
-	case serviceRestart, serviceStop:
-		reply := AuthRPCReply{}
-		err = rc.Call(signalServiceRPC, &SignalServiceArgs{Sig: s},
-			&reply)
-	default:
-		err = errUnsupportedSignal
-	}
-	return err
-}
-
-// ReInitFormat - re-initialize disk format, remotely.
-func (rc remoteAdminClient) ReInitFormat(dryRun bool) error {
-	reply := AuthRPCReply{}
-	return rc.Call(reInitFormatRPC, &ReInitFormatArgs{
-		DryRun: dryRun,
-	}, &reply)
-}
-
-// ListLocks - Sends list locks command to remote server via RPC.
-func (rc remoteAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	listArgs := ListLocksQuery{
-		Bucket:   bucket,
-		Prefix:   prefix,
-		Duration: duration,
-	}
-	var reply ListLocksReply
-	if err := rc.Call(listLocksRPC, &listArgs, &reply); err != nil {
-		return nil, err
-	}
-	return reply.VolLocks, nil
-}
-
-// ServerInfoData - Returns the server info of this server.
-func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	if globalBootTime.IsZero() {
-		return sid, errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return sid, errServerNotInitialized
-	}
-	storage := objLayer.StorageInfo(context.Background())
-
-	return ServerInfoData{
-		StorageInfo: storage,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			SQSARN:   globalNotificationSys.GetARNList(),
-			Region:   globalServerConfig.GetRegion(),
-		},
-	}, nil
-}
-
-// ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	args := AuthRPCArgs{}
-	reply := ServerInfoDataReply{}
-	err := rc.Call(serverInfoDataRPC, &args, &reply)
-	if err != nil {
-		return sid, err
-	}
-
-	return reply.ServerInfoData, nil
-}
-
-// GetConfig - returns config.json of the local server.
-func (lc localAdminClient) GetConfig() ([]byte, error) {
-	if globalServerConfig == nil {
-		return nil, fmt.Errorf("config not present")
-	}
-
-	return json.Marshal(globalServerConfig)
-}
-
-// GetConfig - returns config.json of the remote server.
-func (rc remoteAdminClient) GetConfig() ([]byte, error) {
-	args := AuthRPCArgs{}
-	reply := ConfigReply{}
-	if err := rc.Call(getConfigRPC, &args, &reply); err != nil {
-		return nil, err
-	}
-	return reply.Config, nil
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// the local server.
-func (lc localAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	return writeTmpConfigCommon(tmpFileName, configBytes)
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// a remote node.
-func (rc remoteAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	wArgs := WriteConfigArgs{
-		TmpFileName: tmpFileName,
-		Buf:         configBytes,
-	}
-
-	err := rc.Call(writeTmpConfigRPC, &wArgs, &WriteConfigReply{})
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-
-	return nil
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a local node.
-func (lc localAdminClient) CommitConfig(tmpFileName string) error {
-	configFile := getConfigFile()
-	tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName)
-
-	err := os.Rename(tmpConfigFile, configFile)
-	reqInfo := (&logger.ReqInfo{}).AppendTags("tmpConfigFile", tmpConfigFile)
-	reqInfo.AppendTags("configFile", configFile)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
-	logger.LogIf(ctx, err)
-	return err
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a remote node.
-func (rc remoteAdminClient) CommitConfig(tmpFileName string) error {
-	cArgs := CommitConfigArgs{
-		FileName: tmpFileName,
-	}
-	cReply := CommitConfigReply{}
-	err := rc.Call(commitConfigRPC, &cArgs, &cReply)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-
-	return nil
-}
-
-// adminPeer - represents an entity that implements admin API RPCs.
-type adminPeer struct {
-	addr      string
-	cmdRunner adminCmdRunner
-	isLocal   bool
-}
-
-// type alias for a collection of adminPeer.
-type adminPeers []adminPeer
-
-// makeAdminPeers - helper function to construct a collection of adminPeer.
-func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
-	thisPeer := globalMinioAddr
-	if globalMinioHost == "" {
-		thisPeer = net.JoinHostPort("localhost", globalMinioPort)
-	}
-	adminPeerList = append(adminPeerList, adminPeer{
-		thisPeer,
-		localAdminClient{},
-		true,
-	})
-
-	hostSet := set.CreateStringSet(globalMinioAddr)
-	cred := globalServerConfig.GetCredential()
-	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
-	for _, host := range GetRemotePeers(endpoints) {
-		if hostSet.Contains(host) {
-			continue
-		}
-		hostSet.Add(host)
-		adminPeerList = append(adminPeerList, adminPeer{
-			addr: host,
-			cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
-				accessKey:       cred.AccessKey,
-				secretKey:       cred.SecretKey,
-				serverAddr:      host,
-				serviceEndpoint: serviceEndpoint,
-				secureConn:      globalIsSSL,
-				serviceName:     "Admin",
-			})},
-		})
-	}
-
-	return adminPeerList
-}
-
-// peersReInitFormat - reinitialize remote object layers to new format.
-func peersReInitFormat(peers adminPeers, dryRun bool) error {
-	errs := make([]error, len(peers))
-
-	// Send ReInitFormat RPC call to all nodes.
-	// for local adminPeer this is a no-op.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			if !peer.isLocal {
-				errs[idx] = peer.cmdRunner.ReInitFormat(dryRun)
-			}
-		}(i, peer)
-	}
-	wg.Wait()
-	return nil
-}
-
-// Initialize global adminPeer collection.
-func initGlobalAdminPeers(endpoints EndpointList) {
-	globalAdminPeers = makeAdminPeers(endpoints)
-}
-
-// invokeServiceCmd - Invoke Restart/Stop command.
-func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
-	switch cmd {
-	case serviceRestart, serviceStop:
-		err = cp.cmdRunner.SignalService(cmd)
-	}
-	return err
-}
-
-// sendServiceCmd - Invoke Restart command on remote peers
-// adminPeer followed by on the local peer.
-func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
-	// Send service command like stop or restart to all remote nodes and finally run on local node.
-	errs := make([]error, len(cps))
-	var wg sync.WaitGroup
-	remotePeers := cps[1:]
-	for i := range remotePeers {
-		wg.Add(1)
-		go func(idx int) {
-			defer wg.Done()
-			// we use idx+1 because remotePeers slice is 1 position shifted w.r.t cps
-			errs[idx+1] = invokeServiceCmd(remotePeers[idx], cmd)
-		}(i)
-	}
-	wg.Wait()
-	errs[0] = invokeServiceCmd(cps[0], cmd)
-}
-
-// listPeerLocksInfo - fetch list of locks held on the given bucket,
-// matching prefix held longer than duration from all peer servers.
-func listPeerLocksInfo(peers adminPeers, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	// Used to aggregate volume lock information from all nodes.
-	allLocks := make([][]VolumeLockInfo, len(peers))
-	errs := make([]error, len(peers))
-	var wg sync.WaitGroup
-	localPeer := peers[0]
-	remotePeers := peers[1:]
-	for i, remotePeer := range remotePeers {
-		wg.Add(1)
-		go func(idx int, remotePeer adminPeer) {
-			defer wg.Done()
-			// `remotePeers` is right-shifted by one position relative to `peers`
-			allLocks[idx], errs[idx] = remotePeer.cmdRunner.ListLocks(bucket, prefix, duration)
-		}(i+1, remotePeer)
-	}
-	wg.Wait()
-	allLocks[0], errs[0] = localPeer.cmdRunner.ListLocks(bucket, prefix, duration)
-
-	// Summarizing errors received for ListLocks RPC across all
-	// nodes.  N B the possible unavailability of quorum in errors
-	// applies only to distributed setup.
-	errCount, err := reduceErrs(errs, []error{})
-	if err != nil {
-		if errCount >= (len(peers)/2 + 1) {
-			return nil, err
-		}
-		return nil, InsufficientReadQuorum{}
-	}
-
-	// Group lock information across nodes by (bucket, object)
-	// pair. For readability only.
-	paramLockMap := make(map[nsParam][]VolumeLockInfo)
-	for _, nodeLocks := range allLocks {
-		for _, lockInfo := range nodeLocks {
-			param := nsParam{
-				volume: lockInfo.Bucket,
-				path:   lockInfo.Object,
-			}
-			paramLockMap[param] = append(paramLockMap[param], lockInfo)
-		}
-	}
-	groupedLockInfos := []VolumeLockInfo{}
-	for _, volLocks := range paramLockMap {
-		groupedLockInfos = append(groupedLockInfos, volLocks...)
-	}
-	return groupedLockInfos, nil
-}
-
-// uptimeSlice - used to sort uptimes in chronological order.
-type uptimeSlice []struct {
-	err    error
-	uptime time.Duration
-}
-
-func (ts uptimeSlice) Len() int {
-	return len(ts)
-}
-
-func (ts uptimeSlice) Less(i, j int) bool {
-	return ts[i].uptime < ts[j].uptime
-}
-
-func (ts uptimeSlice) Swap(i, j int) {
-	ts[i], ts[j] = ts[j], ts[i]
-}
-
-// getPeerUptimes - returns the uptime since the last time read quorum
-// was established on success. Otherwise returns errXLReadQuorum.
-func getPeerUptimes(peers adminPeers) (time.Duration, error) {
-	// In a single node Erasure or FS backend setup the uptime of
-	// the setup is the uptime of the single minio server
-	// instance.
-	if !globalIsDistXL {
-		return UTCNow().Sub(globalBootTime), nil
-	}
-
-	uptimes := make(uptimeSlice, len(peers))
-
-	// Get up time of all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
-			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Sort uptimes in chronological order.
-	sort.Sort(uptimes)
-
-	// Pick the readQuorum'th uptime in chronological order. i.e,
-	// the time at which read quorum was (re-)established.
-	readQuorum := len(uptimes) / 2
-	validCount := 0
-	latestUptime := time.Duration(0)
-	for _, uptime := range uptimes {
-		if uptime.err != nil {
-			logger.LogIf(context.Background(), uptime.err)
-			continue
-		}
-
-		validCount++
-		if validCount >= readQuorum {
-			latestUptime = uptime.uptime
-			break
-		}
-	}
-
-	// Less than readQuorum "Admin.Uptime" RPC call returned
-	// successfully, so read-quorum unavailable.
-	if validCount < readQuorum {
-		return time.Duration(0), InsufficientReadQuorum{}
-	}
-
-	return latestUptime, nil
-}
-
-// getPeerConfig - Fetches config.json from all nodes in the setup and
-// returns the one that occurs in a majority of them.
-func getPeerConfig(peers adminPeers) ([]byte, error) {
-	if !globalIsDistXL {
-		return peers[0].cmdRunner.GetConfig()
-	}
-
-	errs := make([]error, len(peers))
-	configs := make([][]byte, len(peers))
-
-	// Get config from all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			configs[idx], errs[idx] = peer.cmdRunner.GetConfig()
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Find the maximally occurring config among peers in a
-	// distributed setup.
-
-	serverConfigs := make([]serverConfig, len(peers))
-	for i, configBytes := range configs {
-		if errs[i] != nil {
-			continue
-		}
-
-		// Unmarshal the received config files.
-		err := json.Unmarshal(configBytes, &serverConfigs[i])
-		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peers[i].addr)
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-	}
-
-	configJSON, err := getValidServerConfig(serverConfigs, errs)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return nil, err
-	}
-
-	// Return the config.json that was present quorum or more
-	// number of disks.
-	return json.Marshal(configJSON)
-}
-
-// getValidServerConfig - finds the server config that is present in
-// quorum or more number of servers.
-func getValidServerConfig(serverConfigs []serverConfig, errs []error) (scv serverConfig, e error) {
-	// majority-based quorum
-	quorum := len(serverConfigs)/2 + 1
-
-	// Count the number of disks a config.json was found in.
-	configCounter := make([]int, len(serverConfigs))
-
-	// We group equal serverConfigs by the lowest index of the
-	// same value;  e.g, let us take the following serverConfigs
-	// in a 4-node setup,
-	// serverConfigs == [c1, c2, c1, c1]
-	// configCounter == [3, 1, 0, 0]
-	// c1, c2 are the only distinct values that appear.  c1 is
-	// identified by 0, the lowest index it appears in and c2 is
-	// identified by 1. So, we need to find the number of times
-	// each of these distinct values occur.
-
-	// Invariants:
-
-	// 1. At the beginning of the i-th iteration, the number of
-	// unique configurations seen so far is equal to the number of
-	// non-zero counter values in config[:i].
-
-	// 2. At the beginning of the i-th iteration, the sum of
-	// elements of configCounter[:i] is equal to the number of
-	// non-error configurations seen so far.
-
-	// For each of the serverConfig ...
-	for i := range serverConfigs {
-		// Skip nodes where getConfig failed.
-		if errs[i] != nil {
-			continue
-		}
-		// Check if it is equal to any of the configurations
-		// seen so far. If j == i is reached then we have an
-		// unseen configuration.
-		for j := 0; j <= i; j++ {
-			if j < i && configCounter[j] == 0 {
-				// serverConfigs[j] is known to be
-				// equal to a value that was already
-				// seen. See example above for
-				// clarity.
-				continue
-			} else if j < i && serverConfigs[i].ConfigDiff(&serverConfigs[j]) == "" {
-				// serverConfigs[i] is equal to
-				// serverConfigs[j], update
-				// serverConfigs[j]'s counter since it
-				// is the lower index.
-				configCounter[j]++
-				break
-			} else if j == i {
-				// serverConfigs[i] is equal to no
-				// other value seen before. It is
-				// unique so far.
-				configCounter[i] = 1
-				break
-			} // else invariants specified above are violated.
-		}
-	}
-
-	// We find the maximally occurring server config and check if
-	// there is quorum.
-	var configJSON serverConfig
-	maxOccurrence := 0
-	for i, count := range configCounter {
-		if maxOccurrence < count {
-			maxOccurrence = count
-			configJSON = serverConfigs[i]
-		}
-	}
-
-	// If quorum nodes don't agree.
-	if maxOccurrence < quorum {
-		return scv, errXLWriteQuorum
-	}
-
-	return configJSON, nil
-}
-
-// Write config contents into a temporary file on all nodes.
-func writeTmpConfigPeers(peers adminPeers, tmpFileName string, configBytes []byte) []error {
-	// For a single-node minio server setup.
-	if !globalIsDistXL {
-		err := peers[0].cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
-		return []error{err}
-	}
-
-	errs := make([]error, len(peers))
-
-	// Write config into temporary file on all nodes.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			errs[idx] = peer.cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Return bytes written and errors (if any) during writing
-	// temporary config file.
-	return errs
-}
-
-// Move config contents from the given temporary file onto config.json
-// on all nodes.
-func commitConfigPeers(peers adminPeers, tmpFileName string) []error {
-	// For a single-node minio server setup.
-	if !globalIsDistXL {
-		return []error{peers[0].cmdRunner.CommitConfig(tmpFileName)}
-	}
-
-	errs := make([]error, len(peers))
-
-	// Rename temporary config file into configDir/config.json on
-	// all nodes.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			errs[idx] = peer.cmdRunner.CommitConfig(tmpFileName)
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Return errors (if any) received during rename.
-	return errs
-}

cmd/admin-rpc-client_test.go

@@ -1,260 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"encoding/json"
-	"reflect"
-	"testing"
-)
-
-var (
-	config1 = []byte(`{
-	"version": "13",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "us-east-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "debug"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	}
-}
-`)
-	// diff from config1 - amqp.Enable is True
-	config2 = []byte(`{
-	"version": "13",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "us-east-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "debug"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": true,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	}
-}
-`)
-)
-
-// TestGetValidServerConfig - test for getValidServerConfig.
-func TestGetValidServerConfig(t *testing.T) {
-	var c1, c2 serverConfig
-	err := json.Unmarshal(config1, &c1)
-	if err != nil {
-		t.Fatalf("json unmarshal of %s failed: %v", string(config1), err)
-	}
-
-	err = json.Unmarshal(config2, &c2)
-	if err != nil {
-		t.Fatalf("json unmarshal of %s failed: %v", string(config2), err)
-	}
-
-	// Valid config.
-	noErrs := []error{nil, nil, nil, nil}
-	serverConfigs := []serverConfig{c1, c2, c1, c1}
-	validConfig, err := getValidServerConfig(serverConfigs, noErrs)
-	if err != nil {
-		t.Errorf("Expected a valid config but received %v instead", err)
-	}
-
-	if !reflect.DeepEqual(validConfig, c1) {
-		t.Errorf("Expected valid config to be %v but received %v", config1, validConfig)
-	}
-
-	// Invalid config - no quorum.
-	serverConfigs = []serverConfig{c1, c2, c2, c1}
-	_, err = getValidServerConfig(serverConfigs, noErrs)
-	if err != errXLWriteQuorum {
-		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
-	}
-
-	// All errors
-	allErrs := []error{errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound}
-	serverConfigs = []serverConfig{{}, {}, {}, {}}
-	_, err = getValidServerConfig(serverConfigs, allErrs)
-	if err != errXLWriteQuorum {
-		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
-	}
-}

cmd/admin-rpc-server.go

@@ -1,240 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-)
-
-const adminPath = "/admin"
-
-// adminCmd - exports RPC methods for service status, stop and
-// restart commands.
-type adminCmd struct {
-	AuthRPCServer
-}
-
-// SignalServiceArgs - provides the signal argument to SignalService RPC
-type SignalServiceArgs struct {
-	AuthRPCArgs
-	Sig serviceSignal
-}
-
-// ListLocksQuery - wraps ListLocks API's query values to send over RPC.
-type ListLocksQuery struct {
-	AuthRPCArgs
-	Bucket   string
-	Prefix   string
-	Duration time.Duration
-}
-
-// ListLocksReply - wraps ListLocks response over RPC.
-type ListLocksReply struct {
-	AuthRPCReply
-	VolLocks []VolumeLockInfo
-}
-
-// ServerInfoDataReply - wraps the server info response over RPC.
-type ServerInfoDataReply struct {
-	AuthRPCReply
-	ServerInfoData ServerInfoData
-}
-
-// ConfigReply - wraps the server config response over RPC.
-type ConfigReply struct {
-	AuthRPCReply
-	Config []byte // json-marshalled bytes of serverConfigV13
-}
-
-// SignalService - Send a restart or stop signal to the service
-func (s *adminCmd) SignalService(args *SignalServiceArgs, reply *AuthRPCReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	globalServiceSignalCh <- args.Sig
-	return nil
-}
-
-// ReInitFormatArgs - provides dry-run information to re-initialize format.json
-type ReInitFormatArgs struct {
-	AuthRPCArgs
-	DryRun bool
-}
-
-// ReInitFormat - re-init 'format.json'
-func (s *adminCmd) ReInitFormat(args *ReInitFormatArgs, reply *AuthRPCReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	return objectAPI.ReloadFormat(context.Background(), args.DryRun)
-}
-
-// ListLocks - lists locks held by requests handled by this server instance.
-func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error {
-	if err := query.IsAuthenticated(); err != nil {
-		return err
-	}
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	volLocks, err := objectAPI.ListLocks(context.Background(), query.Bucket, query.Prefix, query.Duration)
-	if err != nil {
-		return err
-	}
-	*reply = ListLocksReply{VolLocks: volLocks}
-	return nil
-}
-
-// ServerInfo - returns the server info when object layer was initialized on this server.
-func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	if globalBootTime.IsZero() {
-		return errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return errServerNotInitialized
-	}
-	storageInfo := objLayer.StorageInfo(context.Background())
-
-	reply.ServerInfoData = ServerInfoData{
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			Region:   globalServerConfig.GetRegion(),
-			SQSARN:   globalNotificationSys.GetARNList(),
-		},
-		StorageInfo: storageInfo,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-	}
-
-	return nil
-}
-
-// GetConfig - returns the config.json of this server.
-func (s *adminCmd) GetConfig(args *AuthRPCArgs, reply *ConfigReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	if globalServerConfig == nil {
-		return fmt.Errorf("config not present")
-	}
-
-	jsonBytes, err := json.Marshal(globalServerConfig)
-	if err != nil {
-		return err
-	}
-
-	reply.Config = jsonBytes
-	return nil
-}
-
-// WriteConfigArgs - wraps the bytes to be written and temporary file name.
-type WriteConfigArgs struct {
-	AuthRPCArgs
-	TmpFileName string
-	Buf         []byte
-}
-
-// WriteConfigReply - wraps the result of a writing config into a temporary file.
-// the remote node.
-type WriteConfigReply struct {
-	AuthRPCReply
-}
-
-func writeTmpConfigCommon(tmpFileName string, configBytes []byte) error {
-	tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName)
-	err := ioutil.WriteFile(tmpConfigFile, configBytes, 0666)
-	reqInfo := (&logger.ReqInfo{}).AppendTags("tmpConfigFile", tmpConfigFile)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
-	logger.LogIf(ctx, err)
-	return err
-}
-
-// WriteTmpConfig - writes the supplied config contents onto the
-// supplied temporary file.
-func (s *adminCmd) WriteTmpConfig(wArgs *WriteConfigArgs, wReply *WriteConfigReply) error {
-	if err := wArgs.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	return writeTmpConfigCommon(wArgs.TmpFileName, wArgs.Buf)
-}
-
-// CommitConfigArgs - wraps the config file name that needs to be
-// committed into config.json on this node.
-type CommitConfigArgs struct {
-	AuthRPCArgs
-	FileName string
-}
-
-// CommitConfigReply - represents response to commit of config file on
-// this node.
-type CommitConfigReply struct {
-	AuthRPCReply
-}
-
-// CommitConfig - Renames the temporary file into config.json on this node.
-func (s *adminCmd) CommitConfig(cArgs *CommitConfigArgs, cReply *CommitConfigReply) error {
-	configFile := getConfigFile()
-	tmpConfigFile := filepath.Join(getConfigDir(), cArgs.FileName)
-
-	err := os.Rename(tmpConfigFile, configFile)
-	reqInfo := (&logger.ReqInfo{}).AppendTags("tmpConfigFile", tmpConfigFile)
-	reqInfo.AppendTags("configFile", configFile)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
-	logger.LogIf(ctx, err)
-	return err
-}
-
-// registerAdminRPCRouter - registers RPC methods for service status,
-// stop and restart commands.
-func registerAdminRPCRouter(router *mux.Router) error {
-	adminRPCHandler := &adminCmd{}
-	adminRPCServer := newRPCServer()
-	err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-	adminRouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
-	adminRouter.Path(adminPath).Handler(adminRPCServer)
-	return nil
-}

cmd/admin-rpc-server_test.go

@@ -1,262 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"encoding/json"
-	"os"
-	"testing"
-)
-
-func testAdminCmd(cmd cmdType, t *testing.T) {
-	// reset globals. this is to make sure that the tests are not
-	// affected by modified globals.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatalf("Failed to create test config - %v", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	adminServer := adminCmd{}
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	err = adminServer.Login(&args, &LoginRPCReply{})
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	go func() {
-		// A test signal receiver
-		<-globalServiceSignalCh
-	}()
-
-	sa := SignalServiceArgs{
-		AuthRPCArgs: AuthRPCArgs{AuthToken: token, Version: globalRPCAPIVersion},
-		Sig:         cmd.toServiceSignal(),
-	}
-
-	genReply := AuthRPCReply{}
-	switch cmd {
-	case restartCmd, stopCmd:
-		if err = adminServer.SignalService(&sa, &genReply); err != nil {
-			t.Errorf("restartCmd/stopCmd: Expected: <nil>, got: %v",
-				err)
-		}
-	default:
-		err = adminServer.SignalService(&sa, &genReply)
-		if err != nil && err.Error() != errUnsupportedSignal.Error() {
-			t.Errorf("invalidSignal %s: unexpected error got: %v",
-				cmd, err)
-		}
-	}
-}
-
-// TestAdminRestart - test for Admin.Restart RPC service.
-func TestAdminRestart(t *testing.T) {
-	testAdminCmd(restartCmd, t)
-}
-
-// TestAdminStop - test for Admin.Stop RPC service.
-func TestAdminStop(t *testing.T) {
-	testAdminCmd(stopCmd, t)
-}
-
-// TestAdminStatus - test for Admin.Status RPC service (error case)
-func TestAdminStatus(t *testing.T) {
-	testAdminCmd(statusCmd, t)
-}
-
-// TestReInitFormat - test for Admin.ReInitFormat RPC service.
-func TestReInitFormat(t *testing.T) {
-	// Reset global variables to start afresh.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig("us-east-1")
-	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	// Initializing objectLayer for HealFormatHandler.
-	_, xlDirs, xlErr := initTestXLObjLayer()
-	if xlErr != nil {
-		t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
-	}
-	defer removeRoots(xlDirs)
-
-	// Set globalEndpoints for a single node XL setup.
-	globalEndpoints = mustGetNewEndpointList(xlDirs...)
-
-	// Setup admin rpc server for an XL backend.
-	globalIsXL = true
-	adminServer := adminCmd{}
-
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	err = adminServer.Login(&args, &LoginRPCReply{})
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	authArgs := AuthRPCArgs{
-		AuthToken: token,
-		Version:   globalRPCAPIVersion,
-	}
-	authReply := AuthRPCReply{}
-
-	err = adminServer.ReInitFormat(&ReInitFormatArgs{
-		AuthRPCArgs: authArgs,
-		DryRun:      false,
-	}, &authReply)
-	if err != nil {
-		t.Errorf("Expected to pass, but failed with %v", err)
-	}
-}
-
-// TestGetConfig - Test for GetConfig admin RPC.
-func TestGetConfig(t *testing.T) {
-	// Reset global variables to start afresh.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig("us-east-1")
-	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	adminServer := adminCmd{}
-	creds := globalServerConfig.GetCredential()
-
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	reply := LoginRPCReply{}
-	err = adminServer.Login(&args, &reply)
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	authArgs := AuthRPCArgs{
-		AuthToken: token,
-		Version:   globalRPCAPIVersion,
-	}
-
-	configReply := ConfigReply{}
-
-	err = adminServer.GetConfig(&authArgs, &configReply)
-	if err != nil {
-		t.Errorf("Expected GetConfig to pass but failed with %v", err)
-	}
-
-	var config serverConfigV13
-	err = json.Unmarshal(configReply.Config, &config)
-	if err != nil {
-		t.Errorf("Expected json unmarshal to pass but failed with %v", err)
-	}
-}
-
-// TestWriteAndCommitConfig - test for WriteTmpConfig and CommitConfig
-// RPC handler.
-func TestWriteAndCommitConfig(t *testing.T) {
-	// Reset global variables to start afresh.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig("us-east-1")
-	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	adminServer := adminCmd{}
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	reply := LoginRPCReply{}
-	err = adminServer.Login(&args, &reply)
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	// Write temporary config.
-	buf := []byte("hello")
-	tmpFileName := mustGetUUID()
-	wArgs := WriteConfigArgs{
-		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: token,
-			Version:   globalRPCAPIVersion,
-		},
-		TmpFileName: tmpFileName,
-		Buf:         buf,
-	}
-
-	err = adminServer.WriteTmpConfig(&wArgs, &WriteConfigReply{})
-	if err != nil {
-		t.Fatalf("Failed to write temporary config %v", err)
-	}
-
-	if err != nil {
-		t.Errorf("Expected to succeed but failed %v", err)
-	}
-
-	cArgs := CommitConfigArgs{
-		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: token,
-			Version:   globalRPCAPIVersion,
-		},
-		FileName: tmpFileName,
-	}
-	cReply := CommitConfigReply{}
-
-	err = adminServer.CommitConfig(&cArgs, &cReply)
-	if err != nil {
-		t.Fatalf("Failed to commit config file %v", err)
-	}
-}

cmd/api-datatypes.go

@@ -23,6 +23,8 @@ import (
 const (
 	// Response request id.
 	responseRequestIDKey = "x-amz-request-id"
+	// Deployment id.
+	responseDeploymentIDKey = "x-minio-deployment-id"
 )
 
 // ObjectIdentifier carries key name for the object to delete.

cmd/api-errors_test.go

@@ -17,13 +17,15 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"testing"
 
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/hash"
 )
 
-var toAPIErrorCodeTests = []struct {
+var toAPIErrorTests = []struct {
 	err     error
 	errCode APIErrorCode
 }{
@@ -51,12 +53,11 @@ var toAPIErrorCodeTests = []struct {
 	{err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch},
 
 	// SSE-C errors
-	{err: errInsecureSSERequest, errCode: ErrInsecureSSECustomerRequest},
-	{err: errInvalidSSEAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
-	{err: errMissingSSEKey, errCode: ErrMissingSSECustomerKey},
-	{err: errInvalidSSEKey, errCode: ErrInvalidSSECustomerKey},
-	{err: errMissingSSEKeyMD5, errCode: ErrMissingSSECustomerKeyMD5},
-	{err: errSSEKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch},
+	{err: crypto.ErrInvalidCustomerAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
+	{err: crypto.ErrMissingCustomerKey, errCode: ErrMissingSSECustomerKey},
+	{err: crypto.ErrInvalidCustomerKey, errCode: ErrInvalidSSECustomerKey},
+	{err: crypto.ErrMissingCustomerKeyMD5, errCode: ErrMissingSSECustomerKeyMD5},
+	{err: crypto.ErrCustomerKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch},
 	{err: errObjectTampered, errCode: ErrObjectTampered},
 
 	{err: nil, errCode: ErrNone},
@@ -64,8 +65,9 @@ var toAPIErrorCodeTests = []struct {
 }
 
 func TestAPIErrCode(t *testing.T) {
-	for i, testCase := range toAPIErrorCodeTests {
-		errCode := toAPIErrorCode(testCase.err)
+	ctx := context.Background()
+	for i, testCase := range toAPIErrorTests {
+		errCode := toAPIErrorCode(ctx, testCase.err)
 		if errCode != testCase.errCode {
 			t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)
 		}

cmd/api-headers.go

@@ -24,6 +24,8 @@ import (
 	"net/http"
 	"strconv"
 	"time"
+
+	"github.com/minio/minio/cmd/crypto"
 )
 
 // Returns a hexadecimal representation of time at the
@@ -34,15 +36,16 @@ func mustGetRequestID(t time.Time) string {
 
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
-	// Set unique request ID for each reply.
-	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
-	w.Header().Set("Server", globalServerUserAgent)
+	w.Header().Set("Server", "Minio/"+ReleaseTag)
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
 	if region := globalServerConfig.GetRegion(); region != "" {
 		w.Header().Set("X-Amz-Bucket-Region", region)
 	}
 	w.Header().Set("Accept-Ranges", "bytes")
+
+	// Remove sensitive information
+	crypto.RemoveSensitiveHeaders(w.Header())
 }
 
 // Encodes the response headers into XML format.
@@ -63,13 +66,10 @@ func encodeResponseJSON(response interface{}) []byte {
 }
 
 // Write object header
-func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) {
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec) (err error) {
 	// set common headers
 	setCommonHeaders(w)
 
-	// Set content length.
-	w.Header().Set("Content-Length", strconv.FormatInt(objInfo.Size, 10))
-
 	// Set last modified time.
 	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
 	w.Header().Set("Last-Modified", lastModified)
@@ -87,6 +87,9 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 		w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
 	}
 
+	if !objInfo.Expires.IsZero() {
+		w.Header().Set("Expires", objInfo.Expires.UTC().Format(http.TimeFormat))
+	}
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
 		if hasPrefix(k, ReservedMetadataPrefix) {
@@ -97,11 +100,34 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 		w.Header().Set(k, v)
 	}
 
-	// for providing ranged content
-	if contentRange != nil && contentRange.offsetBegin > -1 {
-		// Override content-length
-		w.Header().Set("Content-Length", strconv.FormatInt(contentRange.getLength(), 10))
-		w.Header().Set("Content-Range", contentRange.String())
-		w.WriteHeader(http.StatusPartialContent)
+	var totalObjectSize int64
+	switch {
+	case crypto.IsEncrypted(objInfo.UserDefined):
+		totalObjectSize, err = objInfo.DecryptedSize()
+		if err != nil {
+			return err
+		}
+	case objInfo.IsCompressed():
+		totalObjectSize = objInfo.GetActualSize()
+		if totalObjectSize < 0 {
+			return errInvalidDecompressedSize
+		}
+	default:
+		totalObjectSize = objInfo.Size
 	}
+
+	// for providing ranged content
+	start, rangeLen, err := rs.GetOffsetLength(totalObjectSize)
+	if err != nil {
+		return err
+	}
+
+	// Set content length.
+	w.Header().Set("Content-Length", strconv.FormatInt(rangeLen, 10))
+	if rs != nil {
+		contentRange := fmt.Sprintf("bytes %d-%d/%d", start, start+rangeLen-1, totalObjectSize)
+		w.Header().Set("Content-Range", contentRange)
+	}
+
+	return nil
 }

cmd/api-resources.go

@@ -22,59 +22,101 @@ import (
 )
 
 // Parse bucket url queries
-func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
-	prefix = values.Get("prefix")
-	marker = values.Get("marker")
-	delimiter = values.Get("delimiter")
+func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
 	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
 	} else {
 		maxkeys = maxObjectList
 	}
+
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse bucket url queries for ListObjects V2.
-func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string) {
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	// The continuation-token cannot be empty.
+	if val, ok := values["continuation-token"]; ok {
+		if len(val[0]) == 0 {
+			errCode = ErrIncorrectContinuationToken
+			return
+		}
+	}
+
+	if values.Get("max-keys") != "" {
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
+	} else {
+		maxkeys = maxObjectList
+	}
+
 	prefix = values.Get("prefix")
 	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
-	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
-	} else {
-		maxkeys = maxObjectList
-	}
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse bucket url queries for ?uploads
-func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
+func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	if values.Get("max-uploads") != "" {
+		var err error
+		if maxUploads, err = strconv.Atoi(values.Get("max-uploads")); err != nil {
+			errCode = ErrInvalidMaxUploads
+			return
+		}
+	} else {
+		maxUploads = maxUploadsList
+	}
+
 	prefix = values.Get("prefix")
 	keyMarker = values.Get("key-marker")
 	uploadIDMarker = values.Get("upload-id-marker")
 	delimiter = values.Get("delimiter")
-	if values.Get("max-uploads") != "" {
-		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
-	} else {
-		maxUploads = maxUploadsList
-	}
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse object url queries
-func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
-	uploadID = values.Get("uploadId")
-	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string, errCode APIErrorCode) {
+	var err error
+	errCode = ErrNone
+
 	if values.Get("max-parts") != "" {
-		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
+		if maxParts, err = strconv.Atoi(values.Get("max-parts")); err != nil {
+			errCode = ErrInvalidMaxParts
+			return
+		}
 	} else {
 		maxParts = maxPartsList
 	}
+
+	if values.Get("part-number-marker") != "" {
+		if partNumberMarker, err = strconv.Atoi(values.Get("part-number-marker")); err != nil {
+			errCode = ErrInvalidPartNumberMarker
+			return
+		}
+	}
+
+	uploadID = values.Get("uploadId")
 	encodingType = values.Get("encoding-type")
 	return
 }

cmd/api-resources_test.go

@@ -29,6 +29,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		fetchOwner                           bool
 		maxKeys                              int
 		encodingType                         string
+		errCode                              APIErrorCode
 	}{
 		{
 			values: url.Values{
@@ -47,6 +48,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      100,
 			encodingType: "gzip",
+			errCode:      ErrNone,
 		},
 		{
 			values: url.Values{
@@ -64,11 +66,34 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      1000,
 			encodingType: "gzip",
+			errCode:      ErrNone,
+		},
+		{
+			values: url.Values{
+				"prefix":             []string{"photos/"},
+				"continuation-token": []string{""},
+				"start-after":        []string{"start-after"},
+				"delimiter":          []string{"/"},
+				"fetch-owner":        []string{"true"},
+				"encoding-type":      []string{"gzip"},
+			},
+			prefix:       "",
+			token:        "",
+			startAfter:   "",
+			delimiter:    "",
+			fetchOwner:   false,
+			maxKeys:      0,
+			encodingType: "",
+			errCode:      ErrIncorrectContinuationToken,
 		},
 	}
 
 	for i, testCase := range testCases {
-		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType := getListObjectsV2Args(testCase.values)
+		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(testCase.values)
+
+		if errCode != testCase.errCode {
+			t.Errorf("Test %d: Expected error code:%d, got %d", i+1, testCase.errCode, errCode)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}
@@ -131,7 +156,10 @@ func TestListObjectsV1Resources(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		prefix, marker, delimiter, maxKeys, encodingType := getListObjectsV1Args(testCase.values)
+		prefix, marker, delimiter, maxKeys, encodingType, argsErr := getListObjectsV1Args(testCase.values)
+		if argsErr != ErrNone {
+			t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}
@@ -173,7 +201,10 @@ func TestGetObjectsResources(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		uploadID, partNumberMarker, maxParts, encodingType := getObjectResources(testCase.values)
+		uploadID, partNumberMarker, maxParts, encodingType, argsErr := getObjectResources(testCase.values)
+		if argsErr != ErrNone {
+			t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
+		}
 		if uploadID != testCase.uploadID {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.uploadID, uploadID)
 		}

cmd/api-response-multipart.go

@@ -1,56 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import "net/http"
-
-// Represents additional fields necessary for ErrPartTooSmall S3 error.
-type completeMultipartAPIError struct {
-	// Proposed size represents uploaded size of the part.
-	ProposedSize int64
-	// Minimum size allowed epresents the minimum size allowed per
-	// part. Defaults to 5MB.
-	MinSizeAllowed int64
-	// Part number of the part which is incorrect.
-	PartNumber int
-	// ETag of the part which is incorrect.
-	PartETag string
-	// Other default XML error responses.
-	APIErrorResponse
-}
-
-// writeErrorResponsePartTooSmall - function is used specifically to
-// construct a proper error response during CompleteMultipartUpload
-// when one of the parts is < 5MB.
-// The requirement comes due to the fact that generic ErrorResponse
-// XML doesn't carry the additional fields required to send this
-// error. So we construct a new type which lies well within the scope
-// of this function.
-func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err PartTooSmall) {
-
-	apiError := getAPIError(toAPIErrorCode(err))
-	// Generate complete multipart error response.
-	errorResponse := getAPIErrorResponse(apiError, r.URL.Path)
-	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
-	encodedErrorResponse := encodeResponse(cmpErrResp)
-
-	// respond with 400 bad request.
-	w.WriteHeader(apiError.HTTPStatusCode)
-	// Write error body.
-	w.Write(encodedErrorResponse)
-	w.(http.Flusher).Flush()
-}

cmd/api-response.go

@@ -17,12 +17,15 @@
 package cmd
 
 import (
+	"context"
 	"encoding/xml"
 	"net/http"
 	"net/url"
 	"path"
+	"strings"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
 )
 
@@ -277,7 +280,7 @@ func getURLScheme(tls bool) string {
 }
 
 // getObjectLocation gets the fully qualified URL of an object.
-func getObjectLocation(r *http.Request, domain, bucket, object string) string {
+func getObjectLocation(r *http.Request, domains []string, bucket, object string) string {
 	// unit tests do not have host set.
 	if r.Host == "" {
 		return path.Clean(r.URL.Path)
@@ -292,13 +295,31 @@ func getObjectLocation(r *http.Request, domain, bucket, object string) string {
 		Scheme: proto,
 	}
 	// If domain is set then we need to use bucket DNS style.
-	if domain != "" {
-		u.Host = bucket + "." + domain
-		u.Path = path.Join(slashSeparator, object)
+	for _, domain := range domains {
+		if strings.Contains(r.Host, domain) {
+			u.Host = bucket + "." + r.Host
+			u.Path = path.Join(slashSeparator, object)
+			break
+		}
 	}
 	return u.String()
 }
 
+// s3EncodeName encodes string in response when encodingType
+// is specified in AWS S3 requests.
+func s3EncodeName(name string, encodingType string) (result string) {
+	// Quick path to exit
+	if encodingType == "" {
+		return name
+	}
+	encodingType = strings.ToLower(encodingType)
+	switch encodingType {
+	case "url":
+		return url.QueryEscape(name)
+	}
+	return name
+}
+
 // generates ListBucketsResponse from array of BucketInfo which can be
 // serialized to match XML and JSON API spec output.
 func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
@@ -321,7 +342,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 }
 
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
-func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
+func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
 	var contents []Object
 	var prefixes []CommonPrefix
 	var owner = Owner{}
@@ -333,7 +354,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 		if object.Name == "" {
 			continue
 		}
-		content.Key = object.Name
+		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
@@ -343,20 +364,20 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 		content.Owner = owner
 		contents = append(contents, content)
 	}
-	// TODO - support EncodingType in xml decoding
 	data.Name = bucket
 	data.Contents = contents
 
-	data.Prefix = prefix
-	data.Marker = marker
-	data.Delimiter = delimiter
+	data.EncodingType = encodingType
+	data.Prefix = s3EncodeName(prefix, encodingType)
+	data.Marker = s3EncodeName(marker, encodingType)
+	data.Delimiter = s3EncodeName(delimiter, encodingType)
 	data.MaxKeys = maxKeys
 
-	data.NextMarker = resp.NextMarker
+	data.NextMarker = s3EncodeName(resp.NextMarker, encodingType)
 	data.IsTruncated = resp.IsTruncated
 	for _, prefix := range resp.Prefixes {
 		var prefixItem = CommonPrefix{}
-		prefixItem.Prefix = prefix
+		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
 		prefixes = append(prefixes, prefixItem)
 	}
 	data.CommonPrefixes = prefixes
@@ -364,7 +385,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
 	var contents []Object
 	var commonPrefixes []CommonPrefix
 	var owner = Owner{}
@@ -379,7 +400,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		if object.Name == "" {
 			continue
 		}
-		content.Key = object.Name
+		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
@@ -389,20 +410,20 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		content.Owner = owner
 		contents = append(contents, content)
 	}
-	// TODO - support EncodingType in xml decoding
 	data.Name = bucket
 	data.Contents = contents
 
-	data.StartAfter = startAfter
-	data.Delimiter = delimiter
-	data.Prefix = prefix
+	data.EncodingType = encodingType
+	data.StartAfter = s3EncodeName(startAfter, encodingType)
+	data.Delimiter = s3EncodeName(delimiter, encodingType)
+	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.MaxKeys = maxKeys
 	data.ContinuationToken = token
 	data.NextContinuationToken = nextToken
 	data.IsTruncated = isTruncated
 	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
-		prefixItem.Prefix = prefix
+		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
 		commonPrefixes = append(commonPrefixes, prefixItem)
 	}
 	data.CommonPrefixes = commonPrefixes
@@ -446,11 +467,10 @@ func generateCompleteMultpartUploadResponse(bucket, key, location, etag string)
 }
 
 // generates ListPartsResponse from ListPartsInfo.
-func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
-	// TODO - support EncodingType in xml decoding
+func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) ListPartsResponse {
 	listPartsResponse := ListPartsResponse{}
 	listPartsResponse.Bucket = partsInfo.Bucket
-	listPartsResponse.Key = partsInfo.Object
+	listPartsResponse.Key = s3EncodeName(partsInfo.Object, encodingType)
 	listPartsResponse.UploadID = partsInfo.UploadID
 	listPartsResponse.StorageClass = globalMinioDefaultStorageClass
 	listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID
@@ -474,29 +494,29 @@ func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
 }
 
 // generates ListMultipartUploadsResponse for given bucket and ListMultipartsInfo.
-func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo) ListMultipartUploadsResponse {
+func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo, encodingType string) ListMultipartUploadsResponse {
 	listMultipartUploadsResponse := ListMultipartUploadsResponse{}
 	listMultipartUploadsResponse.Bucket = bucket
-	listMultipartUploadsResponse.Delimiter = multipartsInfo.Delimiter
+	listMultipartUploadsResponse.Delimiter = s3EncodeName(multipartsInfo.Delimiter, encodingType)
 	listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated
-	listMultipartUploadsResponse.EncodingType = multipartsInfo.EncodingType
-	listMultipartUploadsResponse.Prefix = multipartsInfo.Prefix
-	listMultipartUploadsResponse.KeyMarker = multipartsInfo.KeyMarker
-	listMultipartUploadsResponse.NextKeyMarker = multipartsInfo.NextKeyMarker
+	listMultipartUploadsResponse.EncodingType = encodingType
+	listMultipartUploadsResponse.Prefix = s3EncodeName(multipartsInfo.Prefix, encodingType)
+	listMultipartUploadsResponse.KeyMarker = s3EncodeName(multipartsInfo.KeyMarker, encodingType)
+	listMultipartUploadsResponse.NextKeyMarker = s3EncodeName(multipartsInfo.NextKeyMarker, encodingType)
 	listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads
 	listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker
 	listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker
 	listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes))
 	for index, commonPrefix := range multipartsInfo.CommonPrefixes {
 		listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{
-			Prefix: commonPrefix,
+			Prefix: s3EncodeName(commonPrefix, encodingType),
 		}
 	}
 	listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads))
 	for index, upload := range multipartsInfo.Uploads {
 		newUpload := Upload{}
 		newUpload.UploadID = upload.UploadID
-		newUpload.Key = upload.Object
+		newUpload.Key = s3EncodeName(upload.Object, encodingType)
 		newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong)
 		listMultipartUploadsResponse.Uploads[index] = newUpload
 	}
@@ -565,49 +585,93 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 }
 
 // writeErrorRespone writes error headers
-func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
-	switch errorCode {
-	case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
+func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL, browser bool) {
+	switch err.Code {
+	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set("Retry-After", "120")
+	case "AccessDenied":
+		// The request is from browser and also if browser
+		// is enabled we need to redirect.
+		if browser && globalIsBrowserEnabled {
+			w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
+			w.WriteHeader(http.StatusTemporaryRedirect)
+			return
+		}
 	}
-	apiError := getAPIError(errorCode)
+
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
+		w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
 	encodedErrorResponse := encodeResponse(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
 
-func writeErrorResponseHeadersOnly(w http.ResponseWriter, errorCode APIErrorCode) {
-	apiError := getAPIError(errorCode)
-	writeResponse(w, apiError.HTTPStatusCode, nil, mimeNone)
+func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
+	writeResponse(w, err.HTTPStatusCode, nil, mimeNone)
 }
 
 // writeErrorResponseJSON - writes error response in JSON format;
 // useful for admin APIs.
-func writeErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
-	apiError := getAPIError(errorCode)
+func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
 
 // writeCustomErrorResponseJSON - similar to writeErrorResponseJSON,
 // but accepts the error message directly (this allows messages to be
 // dynamically generated.)
-func writeCustomErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode,
+func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError,
 	errBody string, reqURL *url.URL) {
 
-	apiError := getAPIError(errorCode)
+	reqInfo := logger.GetReqInfo(ctx)
 	errorResponse := APIErrorResponse{
-		Code:      apiError.Code,
-		Message:   errBody,
-		Resource:  reqURL.Path,
-		RequestID: "3L137",
-		HostID:    "3L137",
+		Code:       err.Code,
+		Message:    errBody,
+		Resource:   reqURL.Path,
+		BucketName: reqInfo.BucketName,
+		Key:        reqInfo.ObjectName,
+		RequestID:  w.Header().Get(responseRequestIDKey),
+		HostID:     w.Header().Get(responseDeploymentIDKey),
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
+}
+
+// writeCustomErrorResponseXML - similar to writeErrorResponse,
+// but accepts the error message directly (this allows messages to be
+// dynamically generated.)
+func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err APIError, errBody string, reqURL *url.URL, browser bool) {
+
+	switch err.Code {
+	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
+		// Set retry-after header to indicate user-agents to retry request after 120secs.
+		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+		w.Header().Set("Retry-After", "120")
+	case "AccessDenied":
+		// The request is from browser and also if browser
+		// is enabled we need to redirect.
+		if browser && globalIsBrowserEnabled {
+			w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
+			w.WriteHeader(http.StatusTemporaryRedirect)
+			return
+		}
+	}
+
+	reqInfo := logger.GetReqInfo(ctx)
+	errorResponse := APIErrorResponse{
+		Code:       err.Code,
+		Message:    errBody,
+		Resource:   reqURL.Path,
+		BucketName: reqInfo.BucketName,
+		Key:        reqInfo.ObjectName,
+		RequestID:  w.Header().Get(responseRequestIDKey),
+		HostID:     w.Header().Get(responseDeploymentIDKey),
+	}
+
+	encodedErrorResponse := encodeResponse(errorResponse)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }

cmd/api-response_test.go

@@ -26,7 +26,7 @@ func TestObjectLocation(t *testing.T) {
 	testCases := []struct {
 		request          *http.Request
 		bucket, object   string
-		domain           string
+		domains          []string
 		expectedLocation string
 	}{
 		// Server binding to localhost IP with https.
@@ -80,7 +80,7 @@ func TestObjectLocation(t *testing.T) {
 				Host:   "mys3.bucket.org",
 				Header: map[string][]string{},
 			},
-			domain:           "mys3.bucket.org",
+			domains:          []string{"mys3.bucket.org"},
 			bucket:           "mybucket",
 			object:           "test/1.txt",
 			expectedLocation: "http://mybucket.mys3.bucket.org/test/1.txt",
@@ -92,14 +92,14 @@ func TestObjectLocation(t *testing.T) {
 					"X-Forwarded-Scheme": {httpsScheme},
 				},
 			},
-			domain:           "mys3.bucket.org",
+			domains:          []string{"mys3.bucket.org"},
 			bucket:           "mybucket",
 			object:           "test/1.txt",
 			expectedLocation: "https://mybucket.mys3.bucket.org/test/1.txt",
 		},
 	}
 	for i, testCase := range testCases {
-		gotLocation := getObjectLocation(testCase.request, testCase.domain, testCase.bucket, testCase.object)
+		gotLocation := getObjectLocation(testCase.request, testCase.domains, testCase.bucket, testCase.object)
 		if testCase.expectedLocation != gotLocation {
 			t.Errorf("Test %d: expected %s, got %s", i+1, testCase.expectedLocation, gotLocation)
 		}

cmd/api-router.go

@@ -20,36 +20,32 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
 )
 
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
 	CacheAPI  func() CacheObjectLayer
+	// Returns true of handlers should interpret encryption.
+	EncryptionEnabled func() bool
 }
 
 // registerAPIRouter - registers S3 compatible APIs.
-func registerAPIRouter(router *mux.Router) {
-	var err error
-	var cacheConfig = globalServerConfig.GetCacheConfig()
-	if len(cacheConfig.Drives) > 0 {
-		// initialize the new disk cache objects.
-		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
-		logger.FatalIf(err, "Unable to initialize disk caching")
-	}
-
+func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
 		CacheAPI:  newCacheObjectsFn,
+		EncryptionEnabled: func() bool {
+			return encryptionEnabled
+		},
 	}
 
 	// API Router
 	apiRouter := router.PathPrefix("/").Subrouter()
 	var routers []*mux.Router
-	if globalDomainName != "" {
-		routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
+	for _, domainName := range globalDomainNames {
+		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
@@ -69,6 +65,12 @@ func registerAPIRouter(router *mux.Router) {
 		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
 		// AbortMultipartUpload
 		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// GetObjectACL - this is a dummy call.
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
+		// GetObjectTagging - this is a dummy call.
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
+		// SelectObjectContent
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
 		// GetObject
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
 		// CopyObject
@@ -84,8 +86,31 @@ func registerAPIRouter(router *mux.Router) {
 		// GetBucketPolicy
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
 
+		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
+		// GetBucketCors - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
+		// GetBucketWebsiteHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
+		// GetBucketVersioningHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
+		// GetBucketAccelerateHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
+		// GetBucketRequestPaymentHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
+		// GetBucketLoggingHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
+		// GetBucketLifecycleHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
+		// GetBucketReplicationHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
+		// GetBucketTaggingHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
+		//DeleteBucketWebsiteHandler
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
+		// DeleteBucketTaggingHandler
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
 
 		// GetBucketNotification
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
@@ -106,7 +131,7 @@ func registerAPIRouter(router *mux.Router) {
 		// HeadBucket
 		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
 		// PostPolicy
-		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
+		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
 		// DeleteMultipleObjects
 		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
 		// DeleteBucketPolicy

cmd/auth-handler.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/subtle"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
@@ -27,7 +28,11 @@ import (
 	"net/http"
 	"strings"
 
+	jwtgo "github.com/dgrijalva/jwt-go"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/hash"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -85,6 +90,7 @@ const (
 	authTypeSigned
 	authTypeSignedV2
 	authTypeJWT
+	authTypeSTS
 )
 
 // Get request authentication type.
@@ -103,6 +109,8 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypeJWT
 	} else if isRequestPostPolicySignatureV4(r) {
 		return authTypePostPolicy
+	} else if _, ok := r.URL.Query()["Action"]; ok {
+		return authTypeSTS
 	} else if _, ok := r.Header["Authorization"]; !ok {
 		return authTypeAnonymous
 	}
@@ -111,43 +119,140 @@ func getRequestAuthType(r *http.Request) authType {
 
 // checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
 // It does not accept presigned or JWT or anonymous requests.
-func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
+func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region string) APIErrorCode {
 	s3Err := ErrAccessDenied
-	if getRequestAuthType(r) == authTypeSigned { // we only support V4 (no presign)
-		s3Err = isReqAuthenticated(r, region)
+	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok &&
+		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
+		// We only support admin credentials to access admin APIs.
+
+		var owner bool
+		_, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
+		if s3Err != ErrNone {
+			return s3Err
+		}
+
+		if !owner {
+			return ErrAccessDenied
+		}
+
+		// we only support V4 (no presign) with auth body
+		s3Err = isReqAuthenticated(ctx, r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		ctx := logger.SetReqInfo(ctx, reqInfo)
 		logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description))
 	}
 	return s3Err
 }
 
-func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) APIErrorCode {
-	isOwner := true
-	accountName := globalServerConfig.GetCredential().AccessKey
+// Fetch the security token set by the client.
+func getSessionToken(r *http.Request) (token string) {
+	token = r.Header.Get("X-Amz-Security-Token")
+	if token != "" {
+		return token
+	}
+	return r.URL.Query().Get("X-Amz-Security-Token")
+}
 
+// Fetch claims in the security token returned by the client, doesn't return
+// errors - upon errors the returned claims map will be empty.
+func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
+	claims, _ := getClaimsFromToken(r)
+	return claims
+}
+
+// Fetch claims in the security token returned by the client.
+func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
+	claims := make(map[string]interface{})
+	token := getSessionToken(r)
+	if token == "" {
+		return claims, nil
+	}
+	stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
+		// JWT token for x-amz-security-token is signed with admin
+		// secret key, temporary credentials become invalid if
+		// server admin credentials change. This is done to ensure
+		// that clients cannot decode the token using the temp
+		// secret keys and generate an entirely new claim by essentially
+		// hijacking the policies. We need to make sure that this is
+		// based an admin credential such that token cannot be decoded
+		// on the client side and is treated like an opaque value.
+		return []byte(globalServerConfig.GetCredential().SecretKey), nil
+	}
+	p := &jwtgo.Parser{
+		ValidMethods: []string{
+			jwtgo.SigningMethodHS256.Alg(),
+			jwtgo.SigningMethodHS512.Alg(),
+		},
+	}
+	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
+	if err != nil {
+		return nil, err
+	}
+	if !jtoken.Valid {
+		return nil, errAuthentication
+	}
+	v, ok := claims["accessKey"]
+	if !ok {
+		return nil, errInvalidAccessKeyID
+	}
+	if _, ok = v.(string); !ok {
+		return nil, errInvalidAccessKeyID
+	}
+	return claims, nil
+}
+
+// Fetch claims in the security token returned by the client and validate the token.
+func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
+	token := getSessionToken(r)
+	if token != "" && cred.AccessKey == "" {
+		return nil, ErrNoAccessKey
+	}
+	if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+		return nil, ErrInvalidToken
+	}
+	claims, err := getClaimsFromToken(r)
+	if err != nil {
+		return nil, toAPIErrorCode(context.Background(), err)
+	}
+	return claims, ErrNone
+}
+
+// Check request auth type verifies the incoming http request
+// - validates the request signature
+// - validates the policy action if anonymous tests bucket policies if any,
+//   for authenticated requests validates IAM policies.
+// returns APIErrorCode if any to be replied to the client.
+func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
+	var cred auth.Credentials
+	var owner bool
 	switch getRequestAuthType(r) {
-	case authTypeUnknown:
+	case authTypeUnknown, authTypeStreamingSigned:
 		return ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
-		if errorCode := isReqAuthenticatedV2(r); errorCode != ErrNone {
-			return errorCode
+		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
+			return s3Err
 		}
+		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
 		region := globalServerConfig.GetRegion()
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
 		}
-
-		if errorCode := isReqAuthenticated(r, region); errorCode != ErrNone {
-			return errorCode
+		if s3Err = isReqAuthenticated(ctx, r, region, serviceS3); s3Err != ErrNone {
+			return s3Err
 		}
-	default:
-		isOwner = false
-		accountName = ""
+		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
+	}
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	claims, s3Err := checkClaimsFromToken(r, cred)
+	if s3Err != ErrNone {
+		return s3Err
 	}
 
 	// LocationConstraint is valid only for CreateBucketAction.
@@ -173,17 +278,31 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
 
-	if globalPolicySys.IsAllowed(policy.Args{
-		AccountName:     accountName,
-		Action:          action,
+	if cred.AccessKey == "" {
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          action,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, locationConstraint, ""),
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
+		Action:          iampolicy.Action(action),
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, locationConstraint),
-		IsOwner:         isOwner,
+		ConditionValues: getConditionValues(r, "", cred.AccessKey),
 		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
 	}) {
 		return ErrNone
 	}
-
 	return ErrAccessDenied
 }
 
@@ -195,73 +314,59 @@ func isReqAuthenticatedV2(r *http.Request) (s3Error APIErrorCode) {
 	return doesPresignV2SignatureMatch(r)
 }
 
-func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode) {
-	sha256sum := getContentSha256Cksum(r)
+func reqSignatureV4Verify(r *http.Request, region string, stype serviceType) (s3Error APIErrorCode) {
+	sha256sum := getContentSha256Cksum(r, stype)
 	switch {
 	case isRequestSignatureV4(r):
-		return doesSignatureMatch(sha256sum, r, region)
+		return doesSignatureMatch(sha256sum, r, region, stype)
 	case isRequestPresignedSignatureV4(r):
-		return doesPresignedSignatureMatch(sha256sum, r, region)
+		return doesPresignedSignatureMatch(sha256sum, r, region, stype)
 	default:
 		return ErrAccessDenied
 	}
 }
 
 // Verify if request has valid AWS Signature Version '4'.
-func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
-	if r == nil {
-		return ErrInternalError
-	}
-
-	if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
+func isReqAuthenticated(ctx context.Context, r *http.Request, region string, stype serviceType) (s3Error APIErrorCode) {
+	if errCode := reqSignatureV4Verify(r, region, stype); errCode != ErrNone {
 		return errCode
 	}
 
-	payload, err := ioutil.ReadAll(r.Body)
+	var (
+		err                       error
+		contentMD5, contentSHA256 []byte
+	)
+	// Extract 'Content-Md5' if present.
+	if _, ok := r.Header["Content-Md5"]; ok {
+		contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get("Content-Md5"))
+		if err != nil || len(contentMD5) == 0 {
+			return ErrInvalidDigest
+		}
+	}
+
+	// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
+	// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
+	if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
+		if sha256Sum, ok := r.URL.Query()["X-Amz-Content-Sha256"]; ok && len(sha256Sum) > 0 {
+			contentSHA256, err = hex.DecodeString(sha256Sum[0])
+			if err != nil {
+				return ErrContentSHA256Mismatch
+			}
+		}
+	} else if _, ok := r.Header["X-Amz-Content-Sha256"]; !skipSHA256 && ok {
+		contentSHA256, err = hex.DecodeString(r.Header.Get("X-Amz-Content-Sha256"))
+		if err != nil || len(contentSHA256) == 0 {
+			return ErrContentSHA256Mismatch
+		}
+	}
+
+	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
+	// The verification happens implicit during reading.
+	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return ErrInternalError
-	}
-
-	// Populate back the payload.
-	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
-
-	// Verify Content-Md5, if payload is set.
-	if clntMD5B64, ok := r.Header["Content-Md5"]; ok {
-		if clntMD5B64[0] == "" {
-			return ErrInvalidDigest
-		}
-		md5Sum, err := base64.StdEncoding.Strict().DecodeString(clntMD5B64[0])
-		if err != nil {
-			return ErrInvalidDigest
-		}
-		if !bytes.Equal(md5Sum, getMD5Sum(payload)) {
-			return ErrBadDigest
-		}
-	}
-
-	if skipContentSha256Cksum(r) {
-		return ErrNone
-	}
-
-	// Verify that X-Amz-Content-Sha256 Header == sha256(payload)
-	// If X-Amz-Content-Sha256 header is not sent then we don't calculate/verify sha256(payload)
-	sumHex, ok := r.Header["X-Amz-Content-Sha256"]
-	if isRequestPresignedSignatureV4(r) {
-		sumHex, ok = r.URL.Query()["X-Amz-Content-Sha256"]
-	}
-	if ok {
-		if sumHex[0] == "" {
-			return ErrContentSHA256Mismatch
-		}
-		sum, err := hex.DecodeString(sumHex[0])
-		if err != nil {
-			return ErrContentSHA256Mismatch
-		}
-		if !bytes.Equal(sum, getSHA256Sum(payload)) {
-			return ErrContentSHA256Mismatch
-		}
+		return toAPIErrorCode(ctx, err)
 	}
+	r.Body = ioutil.NopCloser(reader)
 	return ErrNone
 }
 
@@ -301,12 +406,67 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	} else if aType == authTypeJWT {
 		// Validate Authorization header if its valid for JWT request.
-		if !isHTTPRequestValid(r) {
+		if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
 			w.WriteHeader(http.StatusUnauthorized)
 			return
 		}
 		a.handler.ServeHTTP(w, r)
 		return
+	} else if aType == authTypeSTS {
+		a.handler.ServeHTTP(w, r)
+		return
 	}
-	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL)
+	writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
+}
+
+// isPutAllowed - check if PUT operation is allowed on the resource, this
+// call verifies bucket policies and IAM policies, supports multi user
+// checks etc.
+func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request) (s3Err APIErrorCode) {
+	var cred auth.Credentials
+	var owner bool
+	switch atype {
+	case authTypeUnknown:
+		return ErrAccessDenied
+	case authTypeSignedV2, authTypePresignedV2:
+		cred, owner, s3Err = getReqAccessKeyV2(r)
+	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
+		region := globalServerConfig.GetRegion()
+		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
+	}
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	claims, s3Err := checkClaimsFromToken(r, cred)
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	if cred.AccessKey == "" {
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          policy.PutObjectAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, "", ""),
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
+		Action:          policy.PutObjectAction,
+		BucketName:      bucketName,
+		ConditionValues: getConditionValues(r, "", cred.AccessKey),
+		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
+	}) {
+		return ErrNone
+	}
+	return ErrAccessDenied
 }

cmd/auth-handler_test.go

@@ -18,7 +18,9 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -343,11 +345,14 @@ func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int6
 
 // Tests is requested authenticated function, tests replies for s3 errors.
 func TestIsReqAuthenticated(t *testing.T) {
-	path, err := newTestConfig(globalMinioDefaultRegion)
+	objLayer, fsDir, err := prepareFS()
 	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
@@ -361,8 +366,6 @@ func TestIsReqAuthenticated(t *testing.T) {
 		req     *http.Request
 		s3Error APIErrorCode
 	}{
-		// When request is nil, internal error is returned.
-		{nil, ErrInternalError},
 		// When request is unsigned, access denied is returned.
 		{mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrAccessDenied},
 		// Empty Content-Md5 header.
@@ -375,19 +378,27 @@ func TestIsReqAuthenticated(t *testing.T) {
 		{mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrNone},
 	}
 
+	ctx := context.Background()
 	// Validates all testcases.
-	for _, testCase := range testCases {
-		if s3Error := isReqAuthenticated(testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
-			t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
+	for i, testCase := range testCases {
+		s3Error := isReqAuthenticated(ctx, testCase.req, globalServerConfig.GetRegion(), serviceS3)
+		if s3Error != testCase.s3Error {
+			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
+				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
+			}
 		}
 	}
 }
 func TestCheckAdminRequestAuthType(t *testing.T) {
-	path, err := newTestConfig(globalMinioDefaultRegion)
+	objLayer, fsDir, err := prepareFS()
 	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
@@ -405,8 +416,9 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		{Request: mustNewPresignedV2Request("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 		{Request: mustNewPresignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 	}
+	ctx := context.Background()
 	for i, testCase := range testCases {
-		if s3Error := checkAdminRequestAuthType(testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
+		if s3Error := checkAdminRequestAuthType(ctx, testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}

cmd/auth-rpc-client.go

@@ -1,322 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bufio"
-	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"net/rpc"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/minio/minio/cmd/logger"
-)
-
-// Attempt to retry only this many number of times before
-// giving up on the remote RPC entirely.
-const globalAuthRPCRetryThreshold = 1
-
-// authConfig requires to make new AuthRPCClient.
-type authConfig struct {
-	accessKey        string // Access key (like username) for authentication.
-	secretKey        string // Secret key (like Password) for authentication.
-	serverAddr       string // RPC server address.
-	serviceEndpoint  string // Endpoint on the server to make any RPC call.
-	secureConn       bool   // Make TLS connection to RPC server or not.
-	serviceName      string // Service name of auth server.
-	disableReconnect bool   // Disable reconnect on failure or not.
-
-	/// Retry configurable values.
-
-	// Each retry unit multiplicative, measured in time.Duration.
-	// This is the basic unit used for calculating backoffs.
-	retryUnit time.Duration
-	// Maximum retry duration i.e A caller would wait no more than this
-	// duration to continue their loop.
-	retryCap time.Duration
-
-	// Maximum retries an call authRPC client would do for a failed
-	// RPC call.
-	retryAttemptThreshold int
-}
-
-// AuthRPCClient is a authenticated RPC client which does authentication before doing Call().
-type AuthRPCClient struct {
-	sync.RWMutex             // Mutex to lock this object.
-	rpcClient    *rpc.Client // RPC client to make any RPC call.
-	config       authConfig  // Authentication configuration information.
-	authToken    string      // Authentication token.
-	version      semVersion  // RPC version.
-}
-
-// newAuthRPCClient - returns a JWT based authenticated (go) rpc client, which does automatic reconnect.
-func newAuthRPCClient(config authConfig) *AuthRPCClient {
-	// Check if retry params are set properly if not default them.
-	emptyDuration := time.Duration(int64(0))
-	if config.retryUnit == emptyDuration {
-		config.retryUnit = defaultRetryUnit
-	}
-	if config.retryCap == emptyDuration {
-		config.retryCap = defaultRetryCap
-	}
-	if config.retryAttemptThreshold == 0 {
-		config.retryAttemptThreshold = globalAuthRPCRetryThreshold
-	}
-
-	return &AuthRPCClient{
-		config:  config,
-		version: globalRPCAPIVersion,
-	}
-}
-
-// Login a JWT based authentication is performed with rpc server.
-func (authClient *AuthRPCClient) Login() (err error) {
-	// Login should be attempted one at a time.
-	//
-	// The reason for large region lock here is
-	// to avoid two simultaneous login attempts
-	// racing over each other.
-	//
-	// #1 Login() gets the lock proceeds to login.
-	// #2 Login() waits for the unlock to happen
-	//    after login in #1.
-	// #1 Successfully completes login saves the
-	//    newly acquired token.
-	// #2 Successfully gets the lock and proceeds,
-	//    but since we have acquired the token
-	//    already the call quickly returns.
-	authClient.Lock()
-	defer authClient.Unlock()
-
-	// Attempt to login if not logged in already.
-	if authClient.authToken == "" {
-		var authToken string
-		authToken, err = authenticateNode(authClient.config.accessKey, authClient.config.secretKey)
-		if err != nil {
-			return err
-		}
-
-		// Login to authenticate your token.
-		var (
-			loginMethod = authClient.config.serviceName + loginMethodName
-			loginArgs   = LoginRPCArgs{
-				AuthToken:   authToken,
-				Version:     globalRPCAPIVersion,
-				RequestTime: UTCNow(),
-			}
-		)
-
-		// Re-dial after we have disconnected or if its a fresh run.
-		var rpcClient *rpc.Client
-		rpcClient, err = rpcDial(authClient.config.serverAddr, authClient.config.serviceEndpoint, authClient.config.secureConn)
-		if err != nil {
-			return err
-		}
-
-		if err = rpcClient.Call(loginMethod, &loginArgs, &LoginRPCReply{}); err != nil {
-			// Closing the connection here.
-			rpcClient.Close()
-
-			// gob doesn't provide any typed errors for us to reflect
-			// upon, this is the only way to return proper error.
-			if strings.Contains(err.Error(), "gob: wrong type") {
-				return errRPCAPIVersionUnsupported
-			}
-			return err
-		}
-
-		// Initialize rpc client and auth token after a successful login.
-		authClient.authToken = authToken
-		authClient.rpcClient = rpcClient
-	}
-
-	return nil
-}
-
-// call makes a RPC call after logs into the server.
-func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
-	SetAuthToken(authToken string)
-	SetRPCAPIVersion(version semVersion)
-}, reply interface{}) (err error) {
-	if err = authClient.Login(); err != nil {
-		return err
-	} // On successful login, execute RPC call.
-
-	// Set token before the rpc call.
-	authClient.RLock()
-	defer authClient.RUnlock()
-	args.SetAuthToken(authClient.authToken)
-	args.SetRPCAPIVersion(authClient.version)
-
-	// Do an RPC call.
-	return authClient.rpcClient.Call(serviceMethod, args, reply)
-}
-
-// Call executes RPC call till success or globalAuthRPCRetryThreshold on ErrShutdown.
-func (authClient *AuthRPCClient) Call(serviceMethod string, args interface {
-	SetAuthToken(authToken string)
-	SetRPCAPIVersion(version semVersion)
-}, reply interface{}) (err error) {
-
-	// Done channel is used to close any lingering retry routine, as soon
-	// as this function returns.
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
-	for i := range newRetryTimer(authClient.config.retryUnit, authClient.config.retryCap, doneCh) {
-		if err = authClient.call(serviceMethod, args, reply); err == rpc.ErrShutdown {
-			// As connection at server side is closed, close the rpc client.
-			authClient.Close()
-
-			// Retry if reconnect is not disabled.
-			if !authClient.config.disableReconnect {
-				// Retry until threshold reaches.
-				if i < authClient.config.retryAttemptThreshold {
-					continue
-				}
-			}
-		}
-
-		// gob doesn't provide any typed errors for us to reflect
-		// upon, this is the only way to return proper error.
-		if err != nil && strings.Contains(err.Error(), "gob: wrong type") {
-			// Close the rpc client also when the servers have mismatching rpc versions.
-			authClient.Close()
-
-			err = errRPCAPIVersionUnsupported
-		}
-
-		break
-	}
-	return err
-}
-
-// Close closes underlying RPC Client.
-func (authClient *AuthRPCClient) Close() error {
-	authClient.Lock()
-	defer authClient.Unlock()
-
-	if authClient.rpcClient == nil {
-		return nil
-	}
-
-	authClient.authToken = ""
-	return authClient.rpcClient.Close()
-}
-
-// ServerAddr returns the serverAddr (network address) of the connection.
-func (authClient *AuthRPCClient) ServerAddr() string {
-	return authClient.config.serverAddr
-}
-
-// ServiceEndpoint returns the RPC service endpoint of the connection.
-func (authClient *AuthRPCClient) ServiceEndpoint() string {
-	return authClient.config.serviceEndpoint
-}
-
-// default Dial timeout for RPC connections.
-const defaultDialTimeout = 3 * time.Second
-
-// Connect success message required from rpc server.
-const connectSuccessMessage = "200 Connected to Go RPC"
-
-// dial tries to establish a connection to serverAddr in a safe manner.
-// If there is a valid rpc.Cliemt, it returns that else creates a new one.
-func rpcDial(serverAddr, serviceEndpoint string, secureConn bool) (netRPCClient *rpc.Client, err error) {
-	if serverAddr == "" || serviceEndpoint == "" {
-		return nil, errInvalidArgument
-	}
-	d := &net.Dialer{
-		Timeout: defaultDialTimeout,
-	}
-	var conn net.Conn
-	if secureConn {
-		var hostname string
-		if hostname, _, err = net.SplitHostPort(serverAddr); err != nil {
-			return nil, &net.OpError{
-				Op:   "dial-http",
-				Net:  serverAddr + serviceEndpoint,
-				Addr: nil,
-				Err:  fmt.Errorf("Unable to parse server address <%s>/<%s>: %s", serverAddr, serviceEndpoint, err),
-			}
-		}
-		// ServerName in tls.Config needs to be specified to support SNI certificates.
-		conn, err = tls.DialWithDialer(d, "tcp", serverAddr, &tls.Config{
-			ServerName: hostname,
-			RootCAs:    globalRootCAs,
-		})
-	} else {
-		conn, err = d.Dial("tcp", serverAddr)
-	}
-
-	if err != nil {
-		// Print RPC connection errors that are worthy to display in log.
-		switch err.(type) {
-		case x509.HostnameError:
-			reqInfo := (&logger.ReqInfo{}).AppendTags("serverAddr", serverAddr)
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
-			logger.LogIf(ctx, err)
-		}
-
-		return nil, &net.OpError{
-			Op:   "dial-http",
-			Net:  serverAddr + serviceEndpoint,
-			Addr: nil,
-			Err:  err,
-		}
-	}
-
-	// Check for network errors writing over the dialed conn.
-	if _, err = io.WriteString(conn, "CONNECT "+serviceEndpoint+" HTTP/1.0\n\n"); err != nil {
-		conn.Close()
-		return nil, &net.OpError{
-			Op:   "dial-http",
-			Net:  serverAddr + serviceEndpoint,
-			Addr: nil,
-			Err:  err,
-		}
-	}
-
-	// Attempt to read the HTTP response for the HTTP method CONNECT, upon
-	// success return the RPC connection instance.
-	resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{
-		Method: http.MethodConnect,
-	})
-	if err != nil {
-		conn.Close()
-		return nil, &net.OpError{
-			Op:   "dial-http",
-			Net:  serverAddr + serviceEndpoint,
-			Addr: nil,
-			Err:  err,
-		}
-	}
-	if resp.Status != connectSuccessMessage {
-		conn.Close()
-		return nil, fmt.Errorf("Unexpected HTTP response: %s from %s/%s", resp.Status, serverAddr, serviceEndpoint)
-	}
-
-	// Initialize rpc client.
-	return rpc.NewClient(conn), nil
-}

cmd/auth-rpc-client_test.go

@@ -1,138 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/x509"
-	"os"
-	"path"
-	"testing"
-)
-
-// Tests authorized RPC client.
-func TestAuthRPCClient(t *testing.T) {
-	// reset globals.
-	// this is to make sure that the tests are not affected by modified globals.
-	resetTestGlobals()
-
-	authCfg := authConfig{
-		accessKey:       "123",
-		secretKey:       "123",
-		serverAddr:      "localhost:9000",
-		serviceEndpoint: "/rpc/disk",
-		secureConn:      false,
-		serviceName:     "MyPackage",
-	}
-	authRPC := newAuthRPCClient(authCfg)
-	if authRPC.ServerAddr() != authCfg.serverAddr {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServerAddr(), authCfg.serverAddr)
-	}
-	if authRPC.ServiceEndpoint() != authCfg.serviceEndpoint {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServiceEndpoint(), authCfg.serviceEndpoint)
-	}
-	authCfg = authConfig{
-		accessKey:   "123",
-		secretKey:   "123",
-		secureConn:  false,
-		serviceName: "MyPackage",
-	}
-	authRPC = newAuthRPCClient(authCfg)
-	if authRPC.ServerAddr() != authCfg.serverAddr {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServerAddr(), authCfg.serverAddr)
-	}
-	if authRPC.ServiceEndpoint() != authCfg.serviceEndpoint {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServiceEndpoint(), authCfg.serviceEndpoint)
-	}
-}
-
-// Test rpc dial test.
-func TestRPCDial(t *testing.T) {
-	prevRootCAs := globalRootCAs
-	defer func() {
-		globalRootCAs = prevRootCAs
-	}()
-
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	testServer := StartTestServer(t, "")
-	defer testServer.Stop()
-
-	cert, key, err := generateTLSCertKey("127.0.0.1")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Set global root CAs.
-	globalRootCAs = x509.NewCertPool()
-	globalRootCAs.AppendCertsFromPEM(cert)
-
-	testServerTLS := StartTestTLSServer(t, "", cert, key)
-	defer testServerTLS.Stop()
-
-	adminEndpoint := path.Join(minioReservedBucketPath, adminPath)
-	testCases := []struct {
-		serverAddr     string
-		serverEndpoint string
-		success        bool
-		secure         bool
-	}{
-		// Empty server addr should fail.
-		{
-			serverAddr:     "",
-			serverEndpoint: adminEndpoint,
-			success:        false,
-		},
-		// Unexpected server addr should fail.
-		{
-			serverAddr:     "example.com",
-			serverEndpoint: adminEndpoint,
-			success:        false,
-		},
-		// Server addr connects but fails for CONNECT call.
-		{
-			serverAddr:     "example.com:80",
-			serverEndpoint: "/",
-			success:        false,
-		},
-		// Successful connecting to insecure RPC server.
-		{
-			serverAddr:     testServer.Server.Listener.Addr().String(),
-			serverEndpoint: adminEndpoint,
-			success:        true,
-		},
-		// Successful connecting to secure RPC server.
-		{
-			serverAddr:     testServerTLS.Server.Listener.Addr().String(),
-			serverEndpoint: adminEndpoint,
-			success:        true,
-			secure:         true,
-		},
-	}
-	for i, testCase := range testCases {
-		_, err = rpcDial(testCase.serverAddr, testCase.serverEndpoint, testCase.secure)
-		if err != nil && testCase.success {
-			t.Errorf("Test %d: Expected success but found failure instead %s", i+1, err)
-		}
-		if err == nil && !testCase.success {
-			t.Errorf("Test %d: Expected failure but found success instead", i+1)
-		}
-	}
-}

cmd/auth-rpc-server.go

@@ -1,38 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-// Base login method name.  It should be used along with service name.
-const loginMethodName = ".Login"
-
-// AuthRPCServer RPC server authenticates using JWT.
-type AuthRPCServer struct{}
-
-// Login - Handles JWT based RPC login.
-func (b AuthRPCServer) Login(args *LoginRPCArgs, reply *LoginRPCReply) error {
-	// Validate LoginRPCArgs
-	if err := args.IsValid(); err != nil {
-		return err
-	}
-
-	// Return an error if token is not valid.
-	if !isAuthTokenValid(args.AuthToken) {
-		return errAuthentication
-	}
-
-	return nil
-}

cmd/auth-rpc-server_test.go

@@ -1,88 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"os"
-	"testing"
-	"time"
-)
-
-func TestLogin(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatalf("Failed to create test config - %v", err)
-	}
-	defer os.RemoveAll(rootPath)
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ls := AuthRPCServer{}
-	testCases := []struct {
-		args        LoginRPCArgs
-		skewTime    time.Duration
-		expectedErr error
-	}{
-		// Valid case.
-		{
-			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   globalRPCAPIVersion,
-			},
-			skewTime:    0,
-			expectedErr: nil,
-		},
-		// Valid username, password and request time, not version.
-		{
-			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   semVersion{1, 0, 0},
-			},
-			skewTime:    0,
-			expectedErr: errRPCAPIVersionUnsupported,
-		},
-		// Valid username, password and version, not request time
-		{
-			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   globalRPCAPIVersion,
-			},
-			skewTime:    20 * time.Minute,
-			expectedErr: errServerTimeMismatch,
-		},
-		// Invalid token, fails with authentication error
-		{
-			args: LoginRPCArgs{
-				AuthToken: "",
-				Version:   globalRPCAPIVersion,
-			},
-			skewTime:    0,
-			expectedErr: errAuthentication,
-		},
-	}
-	for i, test := range testCases {
-		reply := LoginRPCReply{}
-		test.args.RequestTime = UTCNow().Add(test.skewTime)
-		err := ls.Login(&test.args, &reply)
-		if err != test.expectedErr {
-			t.Errorf("Test %d: Expected error %v but received %v",
-				i+1, test.expectedErr, err)
-		}
-	}
-}

cmd/benchmark-utils_test.go

@@ -22,7 +22,6 @@ import (
 	"io/ioutil"
 	"math"
 	"math/rand"
-	"os"
 	"strconv"
 	"testing"
 
@@ -50,7 +49,6 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
-	metadata := make(map[string]string)
 
 	md5hex := getMD5Hash(textData)
 	sha256hex := ""
@@ -62,7 +60,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < b.N; i++ {
 		// insert the object.
 		objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -97,13 +95,11 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
-	metadata := make(map[string]string)
-	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata)
+	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	md5hex := getMD5Hash(textData)
 	sha256hex := ""
 
 	var textPartData []byte
@@ -120,10 +116,10 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			} else {
 				textPartData = textData[j*partSize:]
 			}
-			md5hex = getMD5Hash([]byte(textPartData))
+			md5hex := getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
 			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
-				mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex))
+				mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -138,12 +134,6 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 
 // creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
 func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
 	if err != nil {
@@ -151,18 +141,13 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
 	}
 	// cleaning up the backend by removing all the directories and files created on function return.
 	defer removeRoots(disks)
+
 	// uses *testing.B and the object Layer to run the benchmark.
 	runPutObjectPartBenchmark(b, objLayer, objSize)
 }
 
 // creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
 func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
 	if err != nil {
@@ -170,18 +155,13 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
 	}
 	// cleaning up the backend by removing all the directories and files created on function return.
 	defer removeRoots(disks)
+
 	// uses *testing.B and the object Layer to run the benchmark.
 	runPutObjectBenchmark(b, objLayer, objSize)
 }
 
 // creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
 func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
 	if err != nil {
@@ -189,6 +169,7 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
 	}
 	// cleaning up the backend by removing all the directories and files created on function return.
 	defer removeRoots(disks)
+
 	// uses *testing.B and the object Layer to run the benchmark.
 	runPutObjectBenchmarkParallel(b, objLayer, objSize)
 }
@@ -196,16 +177,10 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
 // Benchmark utility functions for ObjectLayer.GetObject().
 // Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
 func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -215,7 +190,6 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate etag for the generated data.
 	// etag of the data to written is required as input for PutObject.
 	// PutObject is the functions which writes the data onto the FS/XL backend.
-	metadata := make(map[string]string)
 
 	// get text data generated for number of bytes equal to object size.
 	md5hex := getMD5Hash(textData)
@@ -225,7 +199,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -240,7 +214,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		var buffer = new(bytes.Buffer)
-		err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "")
+		err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "", ObjectOptions{})
 		if err != nil {
 			b.Error(err)
 		}
@@ -255,10 +229,8 @@ func getRandomByte() []byte {
 	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 	// seeding the random number generator.
 	rand.Seed(UTCNow().UnixNano())
-	var b byte
 	// pick a character randomly.
-	b = letterBytes[rand.Intn(len(letterBytes))]
-	return []byte{b}
+	return []byte{letterBytes[rand.Intn(len(letterBytes))]}
 }
 
 // picks a random byte and repeats it to size bytes.
@@ -269,12 +241,6 @@ func generateBytesData(size int) []byte {
 
 // creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
 func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
 	if err != nil {
@@ -282,18 +248,13 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
 	}
 	// cleaning up the backend by removing all the directories and files created.
 	defer removeRoots(disks)
+
 	//  uses *testing.B and the object Layer to run the benchmark.
 	runGetObjectBenchmark(b, objLayer, objSize)
 }
 
 // creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
 func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
 	if err != nil {
@@ -301,6 +262,7 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
 	}
 	// cleaning up the backend by removing all the directories and files created.
 	defer removeRoots(disks)
+
 	//  uses *testing.B and the object Layer to run the benchmark.
 	runGetObjectBenchmarkParallel(b, objLayer, objSize)
 }
@@ -308,16 +270,10 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
 // Parallel benchmark utility functions for ObjectLayer.PutObject().
 // Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
 func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -326,7 +282,6 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
-	metadata := make(map[string]string)
 
 	md5hex := getMD5Hash([]byte(textData))
 	sha256hex := ""
@@ -341,7 +296,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		for pb.Next() {
 			// insert the object.
 			objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-				mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+				mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -359,16 +314,10 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 // Parallel benchmark utility functions for ObjectLayer.GetObject().
 // Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
 func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -378,7 +327,6 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	// PutObject is the functions which writes the data onto the FS/XL backend.
-	metadata := make(map[string]string)
 
 	md5hex := getMD5Hash([]byte(textData))
 	sha256hex := ""
@@ -387,7 +335,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -403,7 +351,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	b.RunParallel(func(pb *testing.PB) {
 		i := 0
 		for pb.Next() {
-			err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "")
+			err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "", ObjectOptions{})
 			if err != nil {
 				b.Error(err)
 			}

cmd/bitrot-streaming.go

@@ -0,0 +1,172 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/hex"
+	"hash"
+	"io"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// Calculates bitrot in chunks and writes the hash into the stream.
+type streamingBitrotWriter struct {
+	iow       *io.PipeWriter
+	h         hash.Hash
+	shardSize int64
+	canClose  chan struct{} // Needed to avoid race explained in Close() call.
+
+	// Following two fields are used only to make sure that Write(p) is called such that
+	// len(p) is always the block size except the last block, i.e prevent programmer errors.
+	currentBlockIdx int
+	verifyTillIdx   int
+}
+
+func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
+	if b.currentBlockIdx < b.verifyTillIdx && int64(len(p)) != b.shardSize {
+		// All blocks except last should be of the length b.shardSize
+		logger.LogIf(context.Background(), errUnexpected)
+		return 0, errUnexpected
+	}
+	if len(p) == 0 {
+		return 0, nil
+	}
+	b.h.Reset()
+	b.h.Write(p)
+	hashBytes := b.h.Sum(nil)
+	n, err := b.iow.Write(hashBytes)
+	if n != len(hashBytes) {
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	n, err = b.iow.Write(p)
+	b.currentBlockIdx++
+	return n, err
+}
+
+func (b *streamingBitrotWriter) Close() error {
+	err := b.iow.Close()
+	// Wait for all data to be written before returning else it causes race conditions.
+	// Race condition is because of io.PipeWriter implementation. i.e consider the following
+	// sequent of operations:
+	// 1) pipe.Write()
+	// 2) pipe.Close()
+	// Now pipe.Close() can return before the data is read on the other end of the pipe and written to the disk
+	// Hence an immediate Read() on the file can return incorrect data.
+	<-b.canClose
+	return err
+}
+
+// Returns streaming bitrot writer implementation.
+func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
+	r, w := io.Pipe()
+	h := algo.New()
+	bw := &streamingBitrotWriter{w, h, shardSize, make(chan struct{}), 0, int(length / shardSize)}
+	go func() {
+		bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
+		totalFileSize := bitrotSumsTotalSize + length
+		err := disk.CreateFile(volume, filePath, totalFileSize, r)
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			r.CloseWithError(err)
+		}
+		close(bw.canClose)
+	}()
+	return bw
+}
+
+// ReadAt() implementation which verifies the bitrot hash available as part of the stream.
+type streamingBitrotReader struct {
+	disk       StorageAPI
+	rc         io.ReadCloser
+	volume     string
+	filePath   string
+	tillOffset int64
+	currOffset int64
+	h          hash.Hash
+	shardSize  int64
+	hashBytes  []byte
+}
+
+func (b *streamingBitrotReader) Close() error {
+	if b.rc == nil {
+		return nil
+	}
+	return b.rc.Close()
+}
+
+func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
+	var err error
+	if offset%b.shardSize != 0 {
+		// Offset should always be aligned to b.shardSize
+		logger.LogIf(context.Background(), errUnexpected)
+		return 0, errUnexpected
+	}
+	if b.rc == nil {
+		// For the first ReadAt() call we need to open the stream for reading.
+		b.currOffset = offset
+		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
+		b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			return 0, err
+		}
+	}
+	if offset != b.currOffset {
+		logger.LogIf(context.Background(), errUnexpected)
+		return 0, errUnexpected
+	}
+	b.h.Reset()
+	_, err = io.ReadFull(b.rc, b.hashBytes)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	_, err = io.ReadFull(b.rc, buf)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	b.h.Write(buf)
+
+	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
+		err = hashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	b.currOffset += int64(len(buf))
+	return len(buf), nil
+}
+
+// Returns streaming bitrot reader implementation.
+func newStreamingBitrotReader(disk StorageAPI, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
+	h := algo.New()
+	return &streamingBitrotReader{
+		disk,
+		nil,
+		volume,
+		filePath,
+		ceilFrac(tillOffset, shardSize)*int64(h.Size()) + tillOffset,
+		0,
+		h,
+		shardSize,
+		make([]byte, h.Size()),
+	}
+}

cmd/bitrot-whole.go

@@ -0,0 +1,109 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"hash"
+	"io"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// Implementation to calculate bitrot for the whole file.
+type wholeBitrotWriter struct {
+	disk      StorageAPI
+	volume    string
+	filePath  string
+	shardSize int64 // This is the shard size of the erasure logic
+	hash.Hash       // For bitrot hash
+
+	// Following two fields are used only to make sure that Write(p) is called such that
+	// len(p) is always the block size except the last block and prevent programmer errors.
+	currentBlockIdx int
+	lastBlockIdx    int
+}
+
+func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
+	if b.currentBlockIdx < b.lastBlockIdx && int64(len(p)) != b.shardSize {
+		// All blocks except last should be of the length b.shardSize
+		logger.LogIf(context.Background(), errUnexpected)
+		return 0, errUnexpected
+	}
+	err := b.disk.AppendFile(b.volume, b.filePath, p)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	_, err = b.Hash.Write(p)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return 0, err
+	}
+	b.currentBlockIdx++
+	return len(p), nil
+}
+
+func (b *wholeBitrotWriter) Close() error {
+	return nil
+}
+
+// Returns whole-file bitrot writer.
+func newWholeBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
+	return &wholeBitrotWriter{disk, volume, filePath, shardSize, algo.New(), 0, int(length / shardSize)}
+}
+
+// Implementation to verify bitrot for the whole file.
+type wholeBitrotReader struct {
+	disk       StorageAPI
+	volume     string
+	filePath   string
+	verifier   *BitrotVerifier // Holds the bit-rot info
+	tillOffset int64           // Affects the length of data requested in disk.ReadFile depending on Read()'s offset
+	buf        []byte          // Holds bit-rot verified data
+}
+
+func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error) {
+	if b.buf == nil {
+		b.buf = make([]byte, b.tillOffset-offset)
+		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
+			ctx := context.Background()
+			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
+			logger.LogIf(ctx, err)
+			return 0, err
+		}
+	}
+	if len(b.buf) < len(buf) {
+		logger.LogIf(context.Background(), errLessData)
+		return 0, errLessData
+	}
+	n = copy(buf, b.buf)
+	b.buf = b.buf[n:]
+	return n, nil
+}
+
+// Returns whole-file bitrot reader.
+func newWholeBitrotReader(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm, tillOffset int64, sum []byte) *wholeBitrotReader {
+	return &wholeBitrotReader{
+		disk:       disk,
+		volume:     volume,
+		filePath:   filePath,
+		verifier:   &BitrotVerifier{algo, sum},
+		tillOffset: tillOffset,
+		buf:        nil,
+	}
+}

cmd/bitrot.go

@@ -0,0 +1,187 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"hash"
+	"io"
+
+	"github.com/minio/highwayhash"
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"golang.org/x/crypto/blake2b"
+)
+
+// magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key.
+var magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0")
+
+// BitrotAlgorithm specifies a algorithm used for bitrot protection.
+type BitrotAlgorithm uint
+
+const (
+	// SHA256 represents the SHA-256 hash function
+	SHA256 BitrotAlgorithm = 1 + iota
+	// HighwayHash256 represents the HighwayHash-256 hash function
+	HighwayHash256
+	// HighwayHash256S represents the Streaming HighwayHash-256 hash function
+	HighwayHash256S
+	// BLAKE2b512 represents the BLAKE2b-512 hash function
+	BLAKE2b512
+)
+
+// DefaultBitrotAlgorithm is the default algorithm used for bitrot protection.
+const (
+	DefaultBitrotAlgorithm = HighwayHash256S
+)
+
+var bitrotAlgorithms = map[BitrotAlgorithm]string{
+	SHA256:          "sha256",
+	BLAKE2b512:      "blake2b",
+	HighwayHash256:  "highwayhash256",
+	HighwayHash256S: "highwayhash256S",
+}
+
+// New returns a new hash.Hash calculating the given bitrot algorithm.
+func (a BitrotAlgorithm) New() hash.Hash {
+	switch a {
+	case SHA256:
+		return sha256.New()
+	case BLAKE2b512:
+		b2, _ := blake2b.New512(nil) // New512 never returns an error if the key is nil
+		return b2
+	case HighwayHash256:
+		hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
+		return hh
+	case HighwayHash256S:
+		hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
+		return hh
+	default:
+		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
+		return nil
+	}
+}
+
+// Available reports whether the given algorithm is available.
+func (a BitrotAlgorithm) Available() bool {
+	_, ok := bitrotAlgorithms[a]
+	return ok
+}
+
+// String returns the string identifier for a given bitrot algorithm.
+// If the algorithm is not supported String panics.
+func (a BitrotAlgorithm) String() string {
+	name, ok := bitrotAlgorithms[a]
+	if !ok {
+		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
+	}
+	return name
+}
+
+// NewBitrotVerifier returns a new BitrotVerifier implementing the given algorithm.
+func NewBitrotVerifier(algorithm BitrotAlgorithm, checksum []byte) *BitrotVerifier {
+	return &BitrotVerifier{algorithm, checksum}
+}
+
+// BitrotVerifier can be used to verify protected data.
+type BitrotVerifier struct {
+	algorithm BitrotAlgorithm
+	sum       []byte
+}
+
+// BitrotAlgorithmFromString returns a bitrot algorithm from the given string representation.
+// It returns 0 if the string representation does not match any supported algorithm.
+// The zero value of a bitrot algorithm is never supported.
+func BitrotAlgorithmFromString(s string) (a BitrotAlgorithm) {
+	for alg, name := range bitrotAlgorithms {
+		if name == s {
+			return alg
+		}
+	}
+	return
+}
+
+func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
+	if algo == HighwayHash256S {
+		return newStreamingBitrotWriter(disk, volume, filePath, length, algo, shardSize)
+	}
+	return newWholeBitrotWriter(disk, volume, filePath, length, algo, shardSize)
+}
+
+func newBitrotReader(disk StorageAPI, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
+	if algo == HighwayHash256S {
+		return newStreamingBitrotReader(disk, bucket, filePath, tillOffset, algo, shardSize)
+	}
+	return newWholeBitrotReader(disk, bucket, filePath, algo, tillOffset, sum)
+}
+
+// Close all the readers.
+func closeBitrotReaders(rs []io.ReaderAt) {
+	for _, r := range rs {
+		if br, ok := r.(*streamingBitrotReader); ok {
+			br.Close()
+		}
+	}
+}
+
+// Close all the writers.
+func closeBitrotWriters(ws []io.Writer) {
+	for _, w := range ws {
+		if bw, ok := w.(*streamingBitrotWriter); ok {
+			bw.Close()
+		}
+	}
+}
+
+// Returns hash sum for whole-bitrot, nil for streaming-bitrot.
+func bitrotWriterSum(w io.Writer) []byte {
+	if bw, ok := w.(*wholeBitrotWriter); ok {
+		return bw.Sum(nil)
+	}
+	return nil
+}
+
+// Verify if a file has bitrot error.
+func bitrotCheckFile(disk StorageAPI, volume string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) (err error) {
+	if algo != HighwayHash256S {
+		buf := []byte{}
+		// For whole-file bitrot we don't need to read the entire file as the bitrot verify happens on the server side even if we read 0-bytes.
+		_, err = disk.ReadFile(volume, filePath, 0, buf, NewBitrotVerifier(algo, sum))
+		return err
+	}
+	buf := make([]byte, shardSize)
+	r := newStreamingBitrotReader(disk, volume, filePath, tillOffset, algo, shardSize)
+	defer closeBitrotReaders([]io.ReaderAt{r})
+	var offset int64
+	for {
+		if offset == tillOffset {
+			break
+		}
+		var n int
+		tmpBuf := buf
+		if int64(len(tmpBuf)) > (tillOffset - offset) {
+			tmpBuf = tmpBuf[:(tillOffset - offset)]
+		}
+		n, err = r.ReadAt(tmpBuf, offset)
+		if err != nil {
+			return err
+		}
+		offset += int64(n)
+	}
+	return nil
+}

cmd/bitrot_test.go

@@ -0,0 +1,84 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"testing"
+)
+
+func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
+	tmpDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	volume := "testvol"
+	filePath := "testfile"
+
+	disk, err := newPosix(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	disk.MakeVol(volume)
+
+	writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10)
+
+	_, err = writer.Write([]byte("aaaaaaaaaa"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, err = writer.Write([]byte("aaaaaaaaaa"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, err = writer.Write([]byte("aaaaaaaaaa"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, err = writer.Write([]byte("aaaaa"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	writer.(io.Closer).Close()
+
+	reader := newBitrotReader(disk, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
+	b := make([]byte, 10)
+	if _, err = reader.ReadAt(b, 0); err != nil {
+		log.Fatal(err)
+	}
+	if _, err = reader.ReadAt(b, 10); err != nil {
+		log.Fatal(err)
+	}
+	if _, err = reader.ReadAt(b, 20); err != nil {
+		log.Fatal(err)
+	}
+	if _, err = reader.ReadAt(b[:5], 30); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func TestAllBitrotAlgorithms(t *testing.T) {
+	for bitrotAlgo := range bitrotAlgorithms {
+		testBitrotReaderWriterAlgo(t, bitrotAlgo)
+	}
+}

cmd/bool-flag.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,11 +21,11 @@ import (
 	"fmt"
 )
 
-// BrowserFlag - wrapper bool type.
-type BrowserFlag bool
+// BoolFlag - wrapper bool type.
+type BoolFlag bool
 
-// String - returns string of BrowserFlag.
-func (bf BrowserFlag) String() string {
+// String - returns string of BoolFlag.
+func (bf BoolFlag) String() string {
 	if bf {
 		return "on"
 	}
@@ -33,20 +33,20 @@ func (bf BrowserFlag) String() string {
 	return "off"
 }
 
-// MarshalJSON - converts BrowserFlag into JSON data.
-func (bf BrowserFlag) MarshalJSON() ([]byte, error) {
+// MarshalJSON - converts BoolFlag into JSON data.
+func (bf BoolFlag) MarshalJSON() ([]byte, error) {
 	return json.Marshal(bf.String())
 }
 
-// UnmarshalJSON - parses given data into BrowserFlag.
-func (bf *BrowserFlag) UnmarshalJSON(data []byte) (err error) {
+// UnmarshalJSON - parses given data into BoolFlag.
+func (bf *BoolFlag) UnmarshalJSON(data []byte) (err error) {
 	var s string
 	if err = json.Unmarshal(data, &s); err == nil {
-		b := BrowserFlag(true)
+		b := BoolFlag(true)
 		if s == "" {
 			// Empty string is treated as valid.
 			*bf = b
-		} else if b, err = ParseBrowserFlag(s); err == nil {
+		} else if b, err = ParseBoolFlag(s); err == nil {
 			*bf = b
 		}
 	}
@@ -54,14 +54,15 @@ func (bf *BrowserFlag) UnmarshalJSON(data []byte) (err error) {
 	return err
 }
 
-// ParseBrowserFlag - parses string into BrowserFlag.
-func ParseBrowserFlag(s string) (bf BrowserFlag, err error) {
-	if s == "on" {
+// ParseBoolFlag - parses string into BoolFlag.
+func ParseBoolFlag(s string) (bf BoolFlag, err error) {
+	switch s {
+	case "on":
 		bf = true
-	} else if s == "off" {
+	case "off":
 		bf = false
-	} else {
-		err = fmt.Errorf("invalid value ‘%s’ for BrowserFlag", s)
+	default:
+		err = fmt.Errorf("invalid value ‘%s’ for BoolFlag", s)
 	}
 
 	return bf, err

cmd/bool-flag_test.go

@@ -21,17 +21,17 @@ import (
 	"testing"
 )
 
-// Test BrowserFlag.String()
-func TestBrowserFlagString(t *testing.T) {
-	var bf BrowserFlag
+// Test BoolFlag.String()
+func TestBoolFlagString(t *testing.T) {
+	var bf BoolFlag
 
 	testCases := []struct {
-		flag           BrowserFlag
+		flag           BoolFlag
 		expectedResult string
 	}{
 		{bf, "off"},
-		{BrowserFlag(true), "on"},
-		{BrowserFlag(false), "off"},
+		{BoolFlag(true), "on"},
+		{BoolFlag(false), "off"},
 	}
 
 	for _, testCase := range testCases {
@@ -42,17 +42,17 @@ func TestBrowserFlagString(t *testing.T) {
 	}
 }
 
-// Test BrowserFlag.MarshalJSON()
-func TestBrowserFlagMarshalJSON(t *testing.T) {
-	var bf BrowserFlag
+// Test BoolFlag.MarshalJSON()
+func TestBoolFlagMarshalJSON(t *testing.T) {
+	var bf BoolFlag
 
 	testCases := []struct {
-		flag           BrowserFlag
+		flag           BoolFlag
 		expectedResult string
 	}{
 		{bf, `"off"`},
-		{BrowserFlag(true), `"on"`},
-		{BrowserFlag(false), `"off"`},
+		{BoolFlag(true), `"on"`},
+		{BoolFlag(false), `"off"`},
 	}
 
 	for _, testCase := range testCases {
@@ -63,27 +63,27 @@ func TestBrowserFlagMarshalJSON(t *testing.T) {
 	}
 }
 
-// Test BrowserFlag.UnmarshalJSON()
-func TestBrowserFlagUnmarshalJSON(t *testing.T) {
+// Test BoolFlag.UnmarshalJSON()
+func TestBoolFlagUnmarshalJSON(t *testing.T) {
 	testCases := []struct {
 		data           []byte
-		expectedResult BrowserFlag
+		expectedResult BoolFlag
 		expectedErr    error
 	}{
-		{[]byte(`{}`), BrowserFlag(false), errors.New("json: cannot unmarshal object into Go value of type string")},
-		{[]byte(`["on"]`), BrowserFlag(false), errors.New("json: cannot unmarshal array into Go value of type string")},
-		{[]byte(`"junk"`), BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
-		{[]byte(`"true"`), BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
-		{[]byte(`"false"`), BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
-		{[]byte(`"ON"`), BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
-		{[]byte(`"OFF"`), BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
-		{[]byte(`""`), BrowserFlag(true), nil},
-		{[]byte(`"on"`), BrowserFlag(true), nil},
-		{[]byte(`"off"`), BrowserFlag(false), nil},
+		{[]byte(`{}`), BoolFlag(false), errors.New("json: cannot unmarshal object into Go value of type string")},
+		{[]byte(`["on"]`), BoolFlag(false), errors.New("json: cannot unmarshal array into Go value of type string")},
+		{[]byte(`"junk"`), BoolFlag(false), errors.New("invalid value ‘junk’ for BoolFlag")},
+		{[]byte(`"true"`), BoolFlag(false), errors.New("invalid value ‘true’ for BoolFlag")},
+		{[]byte(`"false"`), BoolFlag(false), errors.New("invalid value ‘false’ for BoolFlag")},
+		{[]byte(`"ON"`), BoolFlag(false), errors.New("invalid value ‘ON’ for BoolFlag")},
+		{[]byte(`"OFF"`), BoolFlag(false), errors.New("invalid value ‘OFF’ for BoolFlag")},
+		{[]byte(`""`), BoolFlag(true), nil},
+		{[]byte(`"on"`), BoolFlag(true), nil},
+		{[]byte(`"off"`), BoolFlag(false), nil},
 	}
 
 	for _, testCase := range testCases {
-		var flag BrowserFlag
+		var flag BoolFlag
 		err := (&flag).UnmarshalJSON(testCase.data)
 		if testCase.expectedErr == nil {
 			if err != nil {
@@ -101,25 +101,25 @@ func TestBrowserFlagUnmarshalJSON(t *testing.T) {
 	}
 }
 
-// Test ParseBrowserFlag()
-func TestParseBrowserFlag(t *testing.T) {
+// Test ParseBoolFlag()
+func TestParseBoolFlag(t *testing.T) {
 	testCases := []struct {
 		flagStr        string
-		expectedResult BrowserFlag
+		expectedResult BoolFlag
 		expectedErr    error
 	}{
-		{"", BrowserFlag(false), errors.New("invalid value ‘’ for BrowserFlag")},
-		{"junk", BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
-		{"true", BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
-		{"false", BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
-		{"ON", BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
-		{"OFF", BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
-		{"on", BrowserFlag(true), nil},
-		{"off", BrowserFlag(false), nil},
+		{"", BoolFlag(false), errors.New("invalid value ‘’ for BoolFlag")},
+		{"junk", BoolFlag(false), errors.New("invalid value ‘junk’ for BoolFlag")},
+		{"true", BoolFlag(false), errors.New("invalid value ‘true’ for BoolFlag")},
+		{"false", BoolFlag(false), errors.New("invalid value ‘false’ for BoolFlag")},
+		{"ON", BoolFlag(false), errors.New("invalid value ‘ON’ for BoolFlag")},
+		{"OFF", BoolFlag(false), errors.New("invalid value ‘OFF’ for BoolFlag")},
+		{"on", BoolFlag(true), nil},
+		{"off", BoolFlag(false), nil},
 	}
 
 	for _, testCase := range testCases {
-		bf, err := ParseBrowserFlag(testCase.flagStr)
+		bf, err := ParseBoolFlag(testCase.flagStr)
 		if testCase.expectedErr == nil {
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)

cmd/browser-peer-rpc.go

@@ -1,142 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"path"
-	"sync"
-	"time"
-
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-)
-
-// SetAuthPeerArgs - Arguments collection for SetAuth RPC call
-type SetAuthPeerArgs struct {
-	// For Auth
-	AuthRPCArgs
-
-	// New credentials that receiving peer should update to.
-	Creds auth.Credentials
-}
-
-// SetAuthPeer - Update to new credentials sent from a peer Minio
-// server. Since credentials are already validated on the sending
-// peer, here we just persist to file and update in-memory config. All
-// subsequently running isAuthTokenValid() calls will fail, and clients
-// will be forced to re-establish connections. Connections will be
-// re-established only when the sending client has also updated its
-// credentials.
-func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthRPCReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	if !args.Creds.IsValid() {
-		return fmt.Errorf("Invalid credential passed")
-	}
-
-	// Acquire lock before updating global configuration.
-	globalServerConfigMu.Lock()
-	defer globalServerConfigMu.Unlock()
-
-	// Update credentials in memory
-	prevCred := globalServerConfig.SetCredential(args.Creds)
-
-	// Save credentials to config file
-	if err := globalServerConfig.Save(); err != nil {
-		// Save the current creds when failed to update.
-		globalServerConfig.SetCredential(prevCred)
-
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-
-	return nil
-}
-
-// Sends SetAuthPeer RPCs to all peers in the Minio cluster
-func updateCredsOnPeers(creds auth.Credentials) map[string]error {
-	peers := GetRemotePeers(globalEndpoints)
-
-	// Array of errors for each peer
-	errs := make([]error, len(peers))
-	var wg sync.WaitGroup
-
-	serverCred := globalServerConfig.GetCredential()
-	// Launch go routines to send request to each peer in parallel.
-	for ix := range peers {
-		wg.Add(1)
-		go func(ix int) {
-			defer wg.Done()
-
-			// Exclude self to avoid race with
-			// invalidating the RPC token.
-			if peers[ix] == globalMinioAddr {
-				errs[ix] = nil
-				return
-			}
-
-			// Initialize client
-			client := newAuthRPCClient(authConfig{
-				accessKey:       serverCred.AccessKey,
-				secretKey:       serverCred.SecretKey,
-				serverAddr:      peers[ix],
-				secureConn:      globalIsSSL,
-				serviceEndpoint: path.Join(minioReservedBucketPath, browserPeerPath),
-				serviceName:     "BrowserPeer",
-			})
-
-			// Construct RPC call arguments.
-			args := SetAuthPeerArgs{Creds: creds}
-
-			// Make RPC call - we only care about error
-			// response and not the reply.
-			err := client.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{})
-
-			// We try a bit hard (3 attempts with 1 second delay)
-			// to set creds on peers in case of failure.
-			if err != nil {
-				for i := 0; i < 2; i++ {
-					time.Sleep(1 * time.Second) // 1 second delay.
-					err = client.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{})
-					if err == nil {
-						break
-					}
-				}
-			}
-
-			// Send result down the channel
-			errs[ix] = err
-		}(ix)
-	}
-
-	// Wait for requests to complete.
-	wg.Wait()
-
-	// Put errors into map.
-	errsMap := make(map[string]error)
-	for i, err := range errs {
-		if err != nil {
-			errsMap[peers[i]] = err
-		}
-	}
-
-	return errsMap
-}

cmd/browser-peer-rpc_test.go

@@ -1,116 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"path"
-	"testing"
-
-	"github.com/minio/minio/pkg/auth"
-)
-
-// API suite container common to both FS and XL.
-type TestRPCBrowserPeerSuite struct {
-	serverType   string
-	testServer   TestServer
-	testAuthConf authConfig
-}
-
-// Setting up the test suite and starting the Test server.
-func (s *TestRPCBrowserPeerSuite) SetUpSuite(t *testing.T) {
-	s.testServer = StartTestBrowserPeerRPCServer(t, s.serverType)
-	s.testAuthConf = authConfig{
-		serverAddr:      s.testServer.Server.Listener.Addr().String(),
-		accessKey:       s.testServer.AccessKey,
-		secretKey:       s.testServer.SecretKey,
-		serviceEndpoint: path.Join(minioReservedBucketPath, browserPeerPath),
-		serviceName:     "BrowserPeer",
-	}
-}
-
-// TeatDownSuite - called implicitly by after all tests are run in
-// browser peer rpc suite.
-func (s *TestRPCBrowserPeerSuite) TearDownSuite(t *testing.T) {
-	s.testServer.Stop()
-}
-
-func TestBrowserPeerRPC(t *testing.T) {
-	// setup code
-	s := &TestRPCBrowserPeerSuite{serverType: "XL"}
-	s.SetUpSuite(t)
-
-	// run test
-	s.testBrowserPeerRPC(t)
-
-	// teardown code
-	s.TearDownSuite(t)
-}
-
-// Tests for browser peer rpc.
-func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
-	// Construct RPC call arguments.
-	creds, err := auth.CreateCredentials("abcd1", "abcd1234")
-	if err != nil {
-		t.Fatalf("unable to create credential. %v", err)
-	}
-
-	// Validate for invalid token.
-	args := SetAuthPeerArgs{Creds: creds}
-	rclient := newAuthRPCClient(s.testAuthConf)
-	defer rclient.Close()
-	if err = rclient.Login(); err != nil {
-		t.Fatal(err)
-	}
-	rclient.authToken = "garbage"
-	if err = rclient.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{}); err != nil {
-		if err.Error() != errInvalidToken.Error() {
-			t.Fatal(err)
-		}
-	}
-
-	// Validate for successful Peer update.
-	args = SetAuthPeerArgs{Creds: creds}
-	client := newAuthRPCClient(s.testAuthConf)
-	defer client.Close()
-	err = client.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Validate for failure in login handler with previous credentials.
-	rclient = newAuthRPCClient(s.testAuthConf)
-	defer rclient.Close()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	rclient.authToken = token
-	if err = rclient.Login(); err != nil {
-		if err.Error() != errInvalidAccessKeyID.Error() {
-			t.Fatal(err)
-		}
-	}
-
-	token, err = authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	rclient.authToken = token
-	if err = rclient.Login(); err != nil {
-		t.Fatal(err)
-	}
-}

cmd/browser-rpc-router.go

@@ -1,53 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-
-	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-)
-
-// Set up an RPC endpoint that receives browser related calls. The
-// original motivation is for propagating credentials change
-// throughout Minio cluster, initiated from a Minio browser session.
-
-const (
-	browserPeerPath = "/browser/setauth"
-)
-
-// The Type exporting methods exposed for RPC calls.
-type browserPeerAPIHandlers struct {
-	AuthRPCServer
-}
-
-// Register RPC router
-func registerBrowserPeerRPCRouter(router *mux.Router) error {
-	bpHandlers := &browserPeerAPIHandlers{AuthRPCServer{}}
-
-	bpRPCServer := newRPCServer()
-	err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-
-	bpRouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
-	bpRouter.Path(browserPeerPath).Handler(bpRPCServer)
-	return nil
-}

cmd/bucket-handlers-listobjects.go

@@ -18,8 +18,12 @@ package cmd
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/minio/cmd/logger"
+
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -29,12 +33,19 @@ import (
 // - delimiter if set should be equal to '/', otherwise the request is rejected.
 // - marker if set should have a common prefix with 'prefix' param, otherwise
 //   the request is rejected.
-func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIErrorCode {
+func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
 	// Max keys cannot be negative.
 	if maxKeys < 0 {
 		return ErrInvalidMaxKeys
 	}
 
+	if encodingType != "" {
+		// Only url encoding type is supported
+		if strings.ToLower(encodingType) != "url" {
+			return ErrInvalidEncodingMethod
+		}
+	}
+
 	/// Minio special conditions for ListObjects.
 
 	// Verify if delimiter is anything other than '/', which we do not support.
@@ -54,64 +65,76 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 // NOTE: It is recommended that this API to be used for application development.
 // Minio continues to support ListObjectsV1 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListObjectsV2")
+	ctx := newContext(r, w, "ListObjectsV2")
+
+	defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+	urlValues := r.URL.Query()
 
-	// In ListObjectsV2 'continuation-token' is the marker.
-	marker := token
-	// Check if 'continuation-token' is empty.
-	if token == "" {
-		// Then we need to use 'start-after' as marker instead.
-		marker = startAfter
+	// Extract all the listObjectsV2 query params to their native values.
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
+	if errCode != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
+		return
 	}
 
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	listObjectsV2 := objectAPI.ListObjectsV2
 	if api.CacheAPI() != nil {
 		listObjectsV2 = api.CacheAPI().ListObjectsV2
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
-	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, fetchOwner, startAfter)
+	// marshaled into S3 compatible XML header.
+	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	for i := range listObjectsV2Info.Objects {
-		if listObjectsV2Info.Objects[i].IsEncrypted() {
+		var actualSize int64
+		if listObjectsV2Info.Objects[i].IsCompressed() {
+			// Read the decompressed size from the meta.json.
+			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
+			if actualSize < 0 {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
+				return
+			}
+			// Set the info.Size to the actualSize.
+			listObjectsV2Info.Objects[i].Size = actualSize
+		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
+			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
 			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
 	}
 
 	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
-		delimiter, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes)
+		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))
@@ -124,59 +147,72 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 // criteria to return a subset of the objects in a bucket.
 //
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListObjectsV1")
+	ctx := newContext(r, w, "ListObjectsV1")
+
+	defer logger.AuditLog(w, r, "ListObjectsV1", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Extract all the litsObjectsV1 query params to their native values.
-	prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
-
-	// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
-	// does not throw an error.
-	if maxKeys < 0 {
-		writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
-		return
-	} // Validate all the query params before beginning to serve the request.
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+	prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
+	// Validate all the query params before beginning to serve the request.
+	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	listObjects := objectAPI.ListObjects
 	if api.CacheAPI() != nil {
 		listObjects = api.CacheAPI().ListObjects
 	}
+
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
+	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	for i := range listObjectsInfo.Objects {
-		if listObjectsInfo.Objects[i].IsEncrypted() {
+		var actualSize int64
+		if listObjectsInfo.Objects[i].IsCompressed() {
+			// Read the decompressed size from the meta.json.
+			actualSize = listObjectsInfo.Objects[i].GetActualSize()
+			if actualSize < 0 {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
+				return
+			}
+			// Set the info.Size to the actualSize.
+			listObjectsInfo.Objects[i].Size = actualSize
+		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
+			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
 			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
 	}
-
-	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
+	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))

cmd/bucket-handlers.go

@@ -17,8 +17,10 @@
 package cmd
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/xml"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -26,47 +28,89 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"sync"
 
 	"github.com/gorilla/mux"
+
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+// Check if there are buckets on server without corresponding entry in etcd backend and
+// make entries. Here is the general flow
+// - Range over all the available buckets
+// - Check if a bucket has an entry in etcd backend
+// -- If no, make an entry
+// -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
+// -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
+func initFederatorBackend(objLayer ObjectLayer) {
+	b, err := objLayer.ListBuckets(context.Background())
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return
+	}
+
+	g := errgroup.WithNErrs(len(b))
+	for index := range b {
+		index := index
+		g.Go(func() error {
+			r, gerr := globalDNSConfig.Get(b[index].Name)
+			if gerr != nil {
+				if gerr == dns.ErrNoEntriesFound {
+					return globalDNSConfig.Put(b[index].Name)
+				}
+				return gerr
+			}
+			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				// There is already an entry for this bucket, with all IP addresses different. This indicates a bucket name collision. Log an error and continue.
+				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", b[index].Name, globalDomainIPs.ToSlice())
+			}
+			return nil
+		}, index)
+	}
+
+	for _, err := range g.Wait() {
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			return
+		}
+	}
+}
+
 // GetBucketLocationHandler - GET Bucket location.
 // -------------------------
 // This operation returns bucket location.
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketLocation")
+	ctx := newContext(r, w, "GetBucketLocation")
+
+	defer logger.AuditLog(w, r, "GetBucketLocation", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetRLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-	defer bucketLock.RUnlock()
 	getBucketInfo := objectAPI.GetBucketInfo
 	if api.CacheAPI() != nil {
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -93,42 +137,50 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 // uploads in the response.
 //
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListMultipartUploads")
+	ctx := newContext(r, w, "ListMultipartUploads")
+
+	defer logger.AuditLog(w, r, "ListMultipartUploads", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
-	if maxUploads < 0 {
-		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL)
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType, errCode := getBucketMultipartResources(r.URL.Query())
+	if errCode != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
+	if maxUploads < 0 {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMaxUploads), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	if keyMarker != "" {
 		// Marker not common with prefix is not implemented.
 		if !hasPrefix(keyMarker, prefix) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL)
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// generate response
-	response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo)
+	response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo, encodingType)
 	encodedSuccessResponse := encodeResponse(response)
 
 	// write success response.
@@ -140,29 +192,53 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 // This implementation of the GET operation returns a list of all buckets
 // owned by the authenticated sender of the request.
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListBuckets")
+	ctx := newContext(r, w, "ListBuckets")
+
+	defer logger.AuditLog(w, r, "ListBuckets", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	listBuckets := objectAPI.ListBuckets
 
+	listBuckets := objectAPI.ListBuckets
 	if api.CacheAPI() != nil {
 		listBuckets = api.CacheAPI().ListBuckets
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	// Invoke the list buckets.
-	bucketsInfo, err := listBuckets(ctx)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
+	// If etcd, dns federation configured list buckets from etcd.
+	var bucketsInfo []BucketInfo
+	if globalDNSConfig != nil {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil && err != dns.ErrNoEntriesFound {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+		bucketSet := set.NewStringSet()
+		for _, dnsRecord := range dnsBuckets {
+			if bucketSet.Contains(dnsRecord.Key) {
+				continue
+			}
+			bucketsInfo = append(bucketsInfo, BucketInfo{
+				Name:    strings.Trim(dnsRecord.Key, slashSeparator),
+				Created: dnsRecord.CreationDate,
+			})
+			bucketSet.Add(dnsRecord.Key)
+		}
+	} else {
+		// Invoke the list buckets.
+		var err error
+		bucketsInfo, err = listBuckets(ctx)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 
 	// Generate response.
@@ -175,14 +251,16 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 // DeleteMultipleObjectsHandler - deletes multiple objects.
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteMultipleObjects")
+	ctx := newContext(r, w, "DeleteMultipleObjects")
+
+	defer logger.AuditLog(w, r, "DeleteMultipleObjects", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -191,7 +269,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		// In the event access is denied, a 200 response should still be returned
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 		if s3Error != ErrAccessDenied {
-			writeErrorResponse(w, s3Error, r.URL)
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -199,24 +277,30 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Content-Md5 is requied should be set
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(w, ErrMissingContentMD5, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allocate incoming content length bytes.
-	deleteXMLBytes := make([]byte, r.ContentLength)
+	var deleteXMLBytes []byte
+	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
+	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
+		deleteXMLBytes = make([]byte, maxBodySize)
+	} else {
+		deleteXMLBytes = make([]byte, r.ContentLength)
+	}
 
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrInternalError, r.URL)
+		writeErrorResponse(ctx, w, toAdminAPIErr(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -224,7 +308,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedXML, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -232,38 +316,28 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	if globalWORMEnabled {
 		// Not required to check whether given objects exist or not, because
 		// DeleteMultipleObject is always successful irrespective of object existence.
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	var wg = &sync.WaitGroup{} // Allocate a new wait group.
-	var dErrs = make([]error, len(deleteObjects.Objects))
-
-	// Delete all requested objects in parallel.
-	for index, object := range deleteObjects.Objects {
-		wg.Add(1)
-		go func(i int, obj ObjectIdentifier) {
-			defer wg.Done()
-			// If the request is denied access, each item
-			// should be marked as 'AccessDenied'
-			if s3Error == ErrAccessDenied {
-				dErrs[i] = PrefixAccessDenied{
-					Bucket: bucket,
-					Object: obj.ObjectName,
-				}
-				return
-			}
-			deleteObject := objectAPI.DeleteObject
-			if api.CacheAPI() != nil {
-				deleteObject = api.CacheAPI().DeleteObject
-			}
-			dErr := deleteObject(ctx, bucket, obj.ObjectName)
-			if dErr != nil {
-				dErrs[i] = dErr
-			}
-		}(index, object)
+	deleteObject := objectAPI.DeleteObject
+	if api.CacheAPI() != nil {
+		deleteObject = api.CacheAPI().DeleteObject
+	}
+
+	var dErrs = make([]error, len(deleteObjects.Objects))
+	for index, object := range deleteObjects.Objects {
+		// If the request is denied access, each item
+		// should be marked as 'AccessDenied'
+		if s3Error == ErrAccessDenied {
+			dErrs[index] = PrefixAccessDenied{
+				Bucket: bucket,
+				Object: object.ObjectName,
+			}
+			continue
+		}
+		dErrs[index] = deleteObject(ctx, bucket, object.ObjectName)
 	}
-	wg.Wait()
 
 	// Collect deleted objects and errors if any.
 	var deletedObjects []ObjectIdentifier
@@ -281,10 +355,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			deletedObjects = append(deletedObjects, object)
 			continue
 		}
+		apiErr := toAPIError(ctx, err)
 		// Error during delete should be collected separately.
 		deleteErrors = append(deleteErrors, DeleteError{
-			Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
-			Message: errorCodeResponse[toAPIErrorCode(err)].Description,
+			Code:    apiErr.Code,
+			Message: apiErr.Description,
 			Key:     object.ObjectName,
 		})
 	}
@@ -298,7 +373,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Get host and port from Request.RemoteAddr failing which
 	// fill them with empty strings.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -311,10 +386,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			Object: ObjectInfo{
 				Name: dobj.ObjectName,
 			},
-			ReqParams: extractReqParams(r),
-			UserAgent: r.UserAgent(),
-			Host:      host,
-			Port:      port,
+			ReqParams:    extractReqParams(r),
+			RespElements: extractRespElements(w),
+			UserAgent:    r.UserAgent(),
+			Host:         host,
+			Port:         port,
 		})
 	}
 }
@@ -323,11 +399,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 // ----------
 // This implementation of the PUT operation creates a new bucket for authenticated request
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucket")
+	ctx := newContext(r, w, "PutBucket")
+
+	defer logger.AuditLog(w, r, "PutBucket", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -335,35 +413,56 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Validate if location sent by the client is valid, reject
 	// requests which do not follow valid region requirements.
 	if !isValidLocation(location) {
-		writeErrorResponse(w, ErrInvalidRegion, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRegion), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	if globalDNSConfig != nil {
+		if _, err := globalDNSConfig.Get(bucket); err != nil {
+			if err == dns.ErrNoEntriesFound {
+				// Proceed to creating a bucket.
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+					return
+				}
+				if err = globalDNSConfig.Put(bucket); err != nil {
+					objectAPI.DeleteBucket(ctx, bucket)
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+					return
+				}
+
+				// Make sure to add Location information here only for bucket
+				w.Header().Set("Location", getObjectLocation(r, globalDomainNames, bucket, ""))
+
+				writeSuccessResponseHeadersOnly(w)
+				return
+			}
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+
+		}
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBucketAlreadyOwnedByYou), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -378,11 +477,23 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 // This implementation of the POST operation handles object creation with a specified
 // signature policy in multipart/form-data
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PostPolicyBucket")
+	ctx := newContext(r, w, "PostPolicyBucket")
+
+	defer logger.AuditLog(w, r, "PostPolicyBucket", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -391,17 +502,17 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
+	resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
 	if err != nil {
-		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// Make sure that the URL  does not contain object name.
 	if bucket != filepath.Clean(resource[1:]) {
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -410,7 +521,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	reader, err := r.MultipartReader()
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -418,7 +529,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	form, err := reader.ReadForm(maxFormMemory)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -429,13 +540,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if file is provided, error out otherwise.
 	if fileBody == nil {
-		writeErrorResponse(w, ErrPOSTFileRequired, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPOSTFileRequired), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -457,113 +568,135 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if successRedirect != "" {
 		redirectURL, err = url.Parse(successRedirect)
 		if err != nil {
-			writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	// Verify policy signature.
-	apiErr := doesPolicySignatureMatch(formValues)
-	if apiErr != ErrNone {
-		writeErrorResponse(w, apiErr, r.URL)
+	errCode := doesPolicySignatureMatch(formValues)
+	if errCode != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
-	if err != nil {
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
-		return
-	}
-
-	// Make sure formValues adhere to policy restrictions.
-	if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
-		writeErrorResponse(w, apiErr, r.URL)
-		return
-	}
-
-	// Ensure that the object size is within expected range, also the file size
-	// should not exceed the maximum single Put size (5 GiB)
-	lengthRange := postPolicyForm.Conditions.ContentLengthRange
-	if lengthRange.Valid {
-		if fileSize < lengthRange.Min {
-			writeErrorResponse(w, toAPIErrorCode(errDataTooSmall), r.URL)
+	// Handle policy if it is set.
+	if len(policyBytes) > 0 {
+		postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
+		if err != nil {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
-		if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
-			writeErrorResponse(w, toAPIErrorCode(errDataTooLarge), r.URL)
+		// Make sure formValues adhere to policy restrictions.
+		if err = checkPostPolicy(formValues, postPolicyForm); err != nil {
+			writeCustomErrorResponseXML(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), err.Error(), r.URL, guessIsBrowserReq(r))
 			return
 		}
+
+		// Ensure that the object size is within expected range, also the file size
+		// should not exceed the maximum single Put size (5 GiB)
+		lengthRange := postPolicyForm.Conditions.ContentLengthRange
+		if lengthRange.Valid {
+			if fileSize < lengthRange.Min {
+				writeErrorResponse(ctx, w, toAPIError(ctx, errDataTooSmall), r.URL, guessIsBrowserReq(r))
+				return
+			}
+
+			if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
+				writeErrorResponse(ctx, w, toAPIError(ctx, errDataTooLarge), r.URL, guessIsBrowserReq(r))
+				return
+			}
+		}
 	}
 
 	// Extract metadata to be saved from received Form.
-	metadata, err := extractMetadataFromHeader(ctx, formValues)
+	metadata := make(map[string]string)
+	err = extractMetadataFromMap(ctx, formValues, metadata)
 	if err != nil {
-		writeErrorResponse(w, ErrInternalError, r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
+	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
+	rawReader := hashReader
+	pReader := NewPutObjReader(rawReader, nil, nil)
+	var objectEncryptionKey []byte
 
+	// This request header needs to be set prior to setting ObjectOptions
+	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
+		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+	}
+	// get gateway encryption options
+	var opts ObjectOptions
+	opts, err = putOpts(ctx, r, bucket, object, metadata)
+	if err != nil {
+		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
+		return
+	}
 	if objectAPI.IsEncryptionSupported() {
-		if hasSSECustomerHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C requests
+		if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C and SSE-S3 requests
 			var reader io.Reader
 			var key []byte
-			key, err = ParseSSECustomerHeader(formValues)
-			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-				return
+			if crypto.SSEC.IsRequested(formValues) {
+				key, err = ParseSSECustomerHeader(formValues)
+				if err != nil {
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+					return
+				}
 			}
-			reader, err = newEncryptReader(hashReader, key, metadata)
+			reader, objectEncryptionKey, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			info := ObjectInfo{Size: fileSize}
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "") // do not try to verify encrypted content
+			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
+			pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
 		}
 	}
 
-	objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata)
+	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, opts)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	location := getObjectLocation(r, globalDomainName, bucket, object)
+	location := getObjectLocation(r, globalDomainNames, bucket, object)
 	w.Header().Set("ETag", `"`+objInfo.ETag+`"`)
 	w.Header().Set("Location", location)
 
 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedPost,
-		BucketName: objInfo.Bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedPost,
+		BucketName:   objInfo.Bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 
 	if successRedirect != "" {
@@ -597,19 +730,21 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 // have permission to access it. Otherwise, the operation might
 // return responses such as 404 Not Found and 403 Forbidden.
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "HeadBucket")
+	ctx := newContext(r, w, "HeadBucket")
+
+	defer logger.AuditLog(w, r, "HeadBucket", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponseHeadersOnly(w, ErrServerNotInitialized)
+		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrServerNotInitialized))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponseHeadersOnly(w, s3Error)
+		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
 		return
 	}
 
@@ -618,7 +753,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
+		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
 
@@ -627,19 +762,21 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 
 // DeleteBucketHandler - Delete bucket
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteBucket")
+	ctx := newContext(r, w, "DeleteBucket")
+
+	defer logger.AuditLog(w, r, "DeleteBucket", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -649,15 +786,21 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 	// Attempt to delete bucket.
 	if err := deleteBucket(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	globalNotificationSys.RemoveNotification(bucket)
 	globalPolicySys.Remove(bucket)
-	for nerr := range globalNotificationSys.DeleteBucket(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
+	globalNotificationSys.DeleteBucket(ctx, bucket)
+
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			// Deleting DNS entry failed, attempt to create the bucket again.
+			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 
 	// Write success response.

cmd/bucket-handlers_test.go

@@ -84,7 +84,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		// initialize httptest Recorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		// construct HTTP request for Get bucket location.
-		req, err := newTestSignedRequestV4("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, err := newTestSignedRequestV4("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -95,12 +95,12 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
 		}
 		if !bytes.Equal(testCase.locationResponse, rec.Body.Bytes()) && testCase.shouldPass {
-			t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.locationResponse), string(rec.Body.Bytes()))
+			t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.locationResponse), rec.Body.String())
 		}
 		errorResponse := APIErrorResponse{}
 		err = xml.Unmarshal(rec.Body.Bytes(), &errorResponse)
 		if err != nil && !testCase.shouldPass {
-			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, string(rec.Body.Bytes()))
+			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, rec.Body.String())
 		}
 		if errorResponse.Resource != testCase.errorResponse.Resource {
 			t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource)
@@ -116,7 +116,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -131,7 +131,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		errorResponse = APIErrorResponse{}
 		err = xml.Unmarshal(recV2.Body.Bytes(), &errorResponse)
 		if err != nil && !testCase.shouldPass {
-			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, string(recV2.Body.Bytes()))
+			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, recV2.Body.String())
 		}
 		if errorResponse.Resource != testCase.errorResponse.Resource {
 			t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource)
@@ -220,7 +220,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		// construct HTTP request for HEAD bucket.
-		req, err := newTestSignedRequestV4("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, err := newTestSignedRequestV4("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -235,7 +235,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -437,7 +437,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 		// construct HTTP request for List multipart uploads endpoint.
 		u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads)
-		req, gerr := newTestSignedRequestV4("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+		req, gerr := newTestSignedRequestV4("GET", u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if gerr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", i+1, instanceType, gerr)
 		}
@@ -454,7 +454,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 		// construct HTTP request for PUT bucket policy endpoint.
 
 		// verify response for V2 signed HTTP request.
-		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -471,7 +471,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 	// construct HTTP request for List multipart uploads endpoint.
 	u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "")
-	req, err := newTestSignedRequestV4("GET", u, 0, nil, "", "") // Generate an anonymous request.
+	req, err := newTestSignedRequestV4("GET", u, 0, nil, "", "", nil) // Generate an anonymous request.
 	if err != nil {
 		t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", instanceType, err)
 	}
@@ -551,7 +551,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	for i, testCase := range testCases {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
-		req, lerr := newTestSignedRequestV4("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, lerr := newTestSignedRequestV4("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if lerr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListBucketsHandler: <ERROR> %v", i+1, instanceType, lerr)
 		}
@@ -568,7 +568,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 		// construct HTTP request for PUT bucket policy endpoint.
 
 		// verify response for V2 signed HTTP request.
-		reqV2, err := newTestSignedRequestV2("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -625,7 +625,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil)
+		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)
@@ -645,8 +645,8 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	getDeleteErrorList := func(objects []ObjectIdentifier) (deleteErrorList []DeleteError) {
 		for _, obj := range objects {
 			deleteErrorList = append(deleteErrorList, DeleteError{
-				Code:    errorCodeResponse[ErrAccessDenied].Code,
-				Message: errorCodeResponse[ErrAccessDenied].Description,
+				Code:    errorCodes[ErrAccessDenied].Code,
+				Message: errorCodes[ErrAccessDenied].Description,
 				Key:     obj.ObjectName,
 			})
 		}
@@ -745,7 +745,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		// Generate a signed or anonymous request based on the testCase
 		if testCase.accessKey != "" {
 			req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
-				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
+				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey, nil)
 		} else {
 			req, err = newTestRequest("POST", getDeleteMultipleObjectsURL("", bucketName),
 				int64(len(testCase.objects)), bytes.NewReader(testCase.objects))
@@ -785,7 +785,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	nilBucket := "dummy-bucket"
 	nilObject := ""
 
-	nilReq, err := newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "")
+	nilReq, err := newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "", nil)
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}

cmd/bucket-notification-handlers.go

@@ -42,30 +42,32 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 // as per http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
 // It returns empty configuration if its not set.
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketNotification")
+	ctx := newContext(r, w, "GetBucketNotification")
+
+	defer logger.AuditLog(w, r, "GetBucketNotification", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	_, err := objAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -74,17 +76,21 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 	if err != nil {
 		// Ignore errNoSuchNotifications to comply with AWS S3.
 		if err != errNoSuchNotifications {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		nConfig = &event.Config{}
 	}
 
+	// If xml namespace is empty, set a default value before returning.
+	if nConfig.XMLNS == "" {
+		nConfig.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
+	}
+
 	notificationBytes, err := xml.Marshal(nConfig)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -94,16 +100,18 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 // PutBucketNotificationHandler - This HTTP handler stores given notification configuration as per
 // http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucketNotification")
+	ctx := newContext(r, w, "PutBucketNotification")
+
+	defer logger.AuditLog(w, r, "PutBucketNotification", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objectAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -111,45 +119,42 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	bucketName := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// PutBucketNotification always needs a Content-Length.
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	var config *event.Config
 	config, err = event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalServerConfig.GetRegion(), globalNotificationSys.targetList)
 	if err != nil {
-		apiErr := ErrMalformedXML
+		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {
-			apiErr = toAPIErrorCode(err)
+			apiErr = toAPIError(ctx, err)
 		}
 
-		writeErrorResponse(w, apiErr, r.URL)
+		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	if err = saveNotificationConfig(objectAPI, bucketName, config); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	rulesMap := config.ToRulesMap()
 	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
-	for nerr := range globalNotificationSys.PutBucketNotification(bucketName, rulesMap) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.PutBucketNotification(ctx, bucketName, rulesMap)
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -157,24 +162,31 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 // ListenBucketNotificationHandler - This HTTP handler sends events to the connected HTTP client.
 // Client should send prefix/suffix object name to match and events to watch as query parameters.
 func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListenBucketNotification")
+	ctx := newContext(r, w, "ListenBucketNotification")
+
+	defer logger.AuditLog(w, r, "ListenBucketNotification", mustGetClaimsFromToken(r))
 
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
-		return
-	}
-	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
+	if !objAPI.IsNotificationSupported() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if !objAPI.IsListenBucketSupported() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
+		return
+	}
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -182,11 +194,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	var prefix string
 	if len(values["prefix"]) > 1 {
-		writeErrorResponse(w, ErrFilterNamePrefix, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNamePrefix), r.URL, guessIsBrowserReq(r))
+		return
 	}
+
 	if len(values["prefix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -195,11 +209,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	var suffix string
 	if len(values["suffix"]) > 1 {
-		writeErrorResponse(w, ErrFilterNameSuffix, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNameSuffix), r.URL, guessIsBrowserReq(r))
+		return
 	}
+
 	if len(values["suffix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -212,7 +228,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	for _, s := range values["events"] {
 		eventName, err := event.ParseName(s)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -220,49 +236,51 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	}
 
 	if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	host, e := xnet.ParseHost(r.RemoteAddr)
-	logger.CriticalIf(ctx, e)
+	host, err := xnet.ParseHost(r.RemoteAddr)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
-	target, e := target.NewHTTPClientTarget(*host, w)
-	logger.CriticalIf(ctx, e)
+	target, err := target.NewHTTPClientTarget(*host, w)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
 
-	if err := globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
+	if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
 	defer globalNotificationSys.RemoveRulesMap(bucketName, rulesMap)
 
-	thisAddr, e := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-	logger.CriticalIf(ctx, e)
-
-	if err := SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
-		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	errCh := globalNotificationSys.ListenBucketNotification(bucketName, eventNames, pattern, target.ID(), *thisAddr)
-	for nerr := range errCh {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
+	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
+		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
 	}
 
+	globalNotificationSys.ListenBucketNotification(ctx, bucketName, eventNames, pattern, target.ID(), *thisAddr)
+
 	<-target.DoneCh
 
-	if err := RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
+	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 }

cmd/bucket-policy-handlers.go

@@ -38,11 +38,13 @@ const (
 // PutBucketPolicyHandler - This HTTP handler stores given bucket policy configuration as per
 // https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucketPolicy")
+	ctx := newContext(r, w, "PutBucketPolicy")
+
+	defer logger.AuditLog(w, r, "PutBucketPolicy", mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -50,51 +52,48 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Error out if Content-Length is missing.
 	// PutBucketPolicy always needs Content-Length.
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Error out if Content-Length is beyond allowed size.
 	if r.ContentLength > maxBucketPolicySize {
-		writeErrorResponse(w, ErrEntityTooLarge, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Version in policy must not be empty
 	if bucketPolicy.Version == "" {
-		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	globalPolicySys.Set(bucket, *bucketPolicy)
-	for nerr := range globalNotificationSys.SetBucketPolicy(bucket, bucketPolicy) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.SetBucketPolicy(ctx, bucket, bucketPolicy)
 
 	// Success.
 	writeSuccessNoContent(w)
@@ -102,11 +101,13 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 // DeleteBucketPolicyHandler - This HTTP handler removes bucket policy configuration.
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteBucketPolicy")
+	ctx := newContext(r, w, "DeleteBucketPolicy")
+
+	defer logger.AuditLog(w, r, "DeleteBucketPolicy", mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -114,26 +115,23 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	globalPolicySys.Remove(bucket)
-	for nerr := range globalNotificationSys.RemoveBucketPolicy(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.RemoveBucketPolicy(ctx, bucket)
 
 	// Success.
 	writeSuccessNoContent(w)
@@ -141,11 +139,13 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 
 // GetBucketPolicyHandler - This HTTP handler returns bucket policy configuration.
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketPolicy")
+	ctx := newContext(r, w, "GetBucketPolicy")
+
+	defer logger.AuditLog(w, r, "GetBucketPolicy", mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -153,27 +153,26 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Read bucket access policy.
 	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	policyData, err := json.Marshal(bucketPolicy)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/bucket-policy-handlers_test.go

@@ -254,7 +254,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testCase.bucketName),
-			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey)
+			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -268,7 +268,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testCase.bucketName),
-			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey)
+			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -304,7 +304,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -346,7 +346,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -360,7 +360,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -417,7 +417,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("GET", getGetPolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -437,11 +437,13 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 		if recV4.Code != testCase.expectedRespStatus {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			var expectedPolicy *policy.Policy
+			expectedPolicy, err = policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}
-			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			var gotPolicy *policy.Policy
+			gotPolicy, err = policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}
@@ -454,7 +456,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("GET", getGetPolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -509,7 +511,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("GET", getGetPolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -589,7 +591,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -639,7 +641,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for Delete bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -661,7 +663,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -678,7 +680,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for Delete bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("DELETE", getDeletePolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -712,7 +714,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)

cmd/build-constants.go

@@ -22,6 +22,9 @@ var (
 	// GOPATH - GOPATH value at the time of build.
 	GOPATH = ""
 
+	// GOROOT - GOROOT value at the time of build.
+	GOROOT = ""
+
 	// Go get development tag.
 	goGetTag = "DEVELOPMENT.GOGET"
 

cmd/certs.go

@@ -25,6 +25,8 @@ import (
 	"encoding/pem"
 	"io/ioutil"
 	"os"
+
+	"github.com/minio/minio/pkg/certs"
 )
 
 // TLSPrivateKeyPassword is the environment variable which contains the password used
@@ -66,41 +68,34 @@ func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 }
 
 func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
-	// Get all CA file names.
-	var caFiles []string
+	rootCAs, _ := x509.SystemCertPool()
+	if rootCAs == nil {
+		// In some systems (like Windows) system cert pool is
+		// not supported or no certificates are present on the
+		// system - so we create a new cert pool.
+		rootCAs = x509.NewCertPool()
+	}
+
 	fis, err := readDir(certsCAsDir)
 	if err != nil {
-		return nil, err
+		if err == errFileNotFound {
+			err = nil // Return success if CA's directory is missing.
+		}
+		return rootCAs, err
 	}
+
+	// Load all custom CA files.
 	for _, fi := range fis {
 		// Skip all directories.
 		if hasSuffix(fi, slashSeparator) {
 			continue
 		}
-		// We are only interested in regular files here.
-		caFiles = append(caFiles, pathJoin(certsCAsDir, fi))
-	}
-	if len(caFiles) == 0 {
-		return nil, nil
-	}
-
-	rootCAs, err := x509.SystemCertPool()
-	if err != nil {
-		// In some systems like Windows, system cert pool is not supported.
-		// Hence we create a new cert pool.
-		rootCAs = x509.NewCertPool()
-	}
-
-	// Load custom root CAs for client requests
-	for _, caFile := range caFiles {
-		caCert, err := ioutil.ReadFile(caFile)
+		caCert, err := ioutil.ReadFile(pathJoin(certsCAsDir, fi))
 		if err != nil {
-			return nil, err
+			return rootCAs, err
 		}
-
 		rootCAs.AppendCertsFromPEM(caCert)
 	}
-
 	return rootCAs, nil
 }
 
@@ -135,39 +130,32 @@ func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	if err != nil {
 		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg(err.Error())
 	}
-	return cert, nil
-}
-
-func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
-	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
-		return nil, nil, nil, false, nil
-	}
-
-	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
-		return nil, nil, nil, false, err
-	}
-
-	var cert tls.Certificate
-	if cert, err = loadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
-		return nil, nil, nil, false, err
-	}
 	// Ensure that the private key is not a P-384 or P-521 EC key.
 	// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
 	if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
 		if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
 			if name := pub.Params().Name; name == "P-384" || name == "P-521" { // unfortunately there is no cleaner way to check
-				return nil, nil, nil, false, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
+				return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
 			}
 		}
+	}
+	return cert, nil
+}
 
+func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
+	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
+		return nil, nil, false, nil
 	}
 
-	tlsCert = &cert
+	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
+		return nil, nil, false, err
+	}
 
-	if rootCAs, err = getRootCAs(getCADir()); err != nil {
-		return nil, nil, nil, false, err
+	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
+	if err != nil {
+		return nil, nil, false, err
 	}
 
 	secureConn = true
-	return x509Certs, rootCAs, tlsCert, secureConn, nil
+	return x509Certs, c, secureConn, nil
 }

cmd/certs_test.go

@@ -223,7 +223,8 @@ func TestGetRootCAs(t *testing.T) {
 		certCAsDir  string
 		expectedErr error
 	}{
-		{"nonexistent-dir", errFileNotFound},
+		// ignores non-existent directories.
+		{"nonexistent-dir", nil},
 		// Ignores directories.
 		{dir1, nil},
 		// Ignore empty directory.

cmd/common-main.go

@@ -17,16 +17,25 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"errors"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
+	etcd "github.com/coreos/etcd/clientv3"
+	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
+	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/target/console"
+	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Check for updates and print a notification message
@@ -41,54 +50,136 @@ func checkUpdate(mode string) {
 	}
 }
 
-func initConfig() {
-	// Config file does not exist, we create it fresh and return upon success.
-	if isFile(getConfigFile()) {
-		logger.FatalIf(migrateConfig(), "Config migration failed")
-		logger.FatalIf(loadConfig(), "Unable to load the configuration file")
-	} else {
-		logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time")
-		logger.Info("Created minio configuration file successfully at " + getConfigDir())
+// Load logger targets based on user's configuration
+func loadLoggers() {
+	auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
+	if ok {
+		// Enable audit HTTP logging through ENV.
+		logger.AddAuditTarget(http.New(auditEndpoint, NewCustomHTTPTransport()))
 	}
+
+	loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
+	if ok {
+		// Enable HTTP logging through ENV.
+		logger.AddTarget(http.New(loggerEndpoint, NewCustomHTTPTransport()))
+	} else {
+		for _, l := range globalServerConfig.Logger.HTTP {
+			if l.Enabled {
+				// Enable http logging
+				logger.AddTarget(http.New(l.Endpoint, NewCustomHTTPTransport()))
+			}
+		}
+	}
+
+	if globalServerConfig.Logger.Console.Enabled {
+		// Enable console logging
+		logger.AddTarget(console.New())
+	}
+
+}
+
+func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
+	var dir string
+	var dirSet bool
+
+	switch {
+	case ctx.IsSet(option):
+		dir = ctx.String(option)
+		dirSet = true
+	case ctx.GlobalIsSet(option):
+		dir = ctx.GlobalString(option)
+		dirSet = true
+		// cli package does not expose parent's option option.  Below code is workaround.
+		if dir == "" || dir == getDefaultDir() {
+			dirSet = false // Unset to false since GlobalIsSet() true is a false positive.
+			if ctx.Parent().GlobalIsSet(option) {
+				dir = ctx.Parent().GlobalString(option)
+				dirSet = true
+			}
+		}
+	default:
+		// Neither local nor global option is provided.  In this case, try to use
+		// default directory.
+		dir = getDefaultDir()
+		if dir == "" {
+			logger.FatalIf(errInvalidArgument, "%s option must be provided", option)
+		}
+	}
+
+	if dir == "" {
+		logger.FatalIf(errors.New("empty directory"), "%s directory cannot be empty", option)
+	}
+
+	// Disallow relative paths, figure out absolute paths.
+	dirAbs, err := filepath.Abs(dir)
+	logger.FatalIf(err, "Unable to fetch absolute path for %s=%s", option, dir)
+
+	logger.FatalIf(mkdirAllIgnorePerm(dirAbs), "Unable to create directory specified %s=%s", option, dir)
+
+	return &ConfigDir{path: dirAbs}, dirSet
 }
 
 func handleCommonCmdArgs(ctx *cli.Context) {
 
-	var configDir string
-
-	if ctx.IsSet("config-dir") {
-		configDir = ctx.String("config-dir")
-	} else if ctx.GlobalIsSet("config-dir") {
-		configDir = ctx.GlobalString("config-dir")
-		// cli package does not expose parent's "config-dir" option.  Below code is workaround.
-		if configDir == "" || configDir == getConfigDir() {
-			if ctx.Parent().GlobalIsSet("config-dir") {
-				configDir = ctx.Parent().GlobalString("config-dir")
-			}
-		}
-	} else {
-		// Neither local nor global config-dir option is provided.  In this case, try to use
-		// default config directory.
-		configDir = getConfigDir()
-		if configDir == "" {
-			logger.FatalIf(errors.New("missing option"), "config-dir option must be provided")
-		}
+	// Get "json" flag from command line argument and
+	// enable json and quite modes if json flag is turned on.
+	globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
+	if globalCLIContext.JSON {
+		logger.EnableJSON()
 	}
 
-	if configDir == "" {
-		logger.FatalIf(errors.New("empty directory"), "Configuration directory cannot be empty")
+	// Get quiet flag from command line argument.
+	globalCLIContext.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
+	if globalCLIContext.Quiet {
+		logger.EnableQuiet()
 	}
 
-	// Disallow relative paths, figure out absolute paths.
-	configDirAbs, err := filepath.Abs(configDir)
-	logger.FatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
-	setConfigDir(configDirAbs)
+	// Get anonymous flag from command line argument.
+	globalCLIContext.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous")
+	if globalCLIContext.Anonymous {
+		logger.EnableAnonymous()
+	}
+
+	// Fetch address option
+	globalCLIContext.Addr = ctx.GlobalString("address")
+	if globalCLIContext.Addr == "" || globalCLIContext.Addr == ":"+globalMinioDefaultPort {
+		globalCLIContext.Addr = ctx.String("address")
+	}
+
+	// Set all config, certs and CAs directories.
+	var configSet, certsSet bool
+	globalConfigDir, configSet = newConfigDirFromCtx(ctx, "config-dir", defaultConfigDir.Get)
+	globalCertsDir, certsSet = newConfigDirFromCtx(ctx, "certs-dir", defaultCertsDir.Get)
+
+	// Remove this code when we deprecate and remove config-dir.
+	// This code is to make sure we inherit from the config-dir
+	// option if certs-dir is not provided.
+	if !certsSet && configSet {
+		globalCertsDir = &ConfigDir{path: filepath.Join(globalConfigDir.Get(), certsDir)}
+	}
+
+	globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)}
+
+	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
+}
+
+// Parses the given compression exclude list `extensions` or `content-types`.
+func parseCompressIncludes(includes []string) ([]string, error) {
+	for _, e := range includes {
+		if len(e) == 0 {
+			return nil, uiErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
+		}
+	}
+	return includes, nil
 }
 
 func handleCommonEnvVars() {
+	compressEnvDelimiter := ","
 	// Start profiler if env is set.
 	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
-		globalProfiler = startProfiler(profiler)
+		var err error
+		globalProfiler, err = startProfiler(profiler, "")
+		logger.FatalIf(err, "Unable to setup a profiler")
 	}
 
 	accessKey := os.Getenv("MINIO_ACCESS_KEY")
@@ -98,6 +189,7 @@ func handleCommonEnvVars() {
 		if err != nil {
 			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
 		}
+		cred.Expiration = timeSentinel
 
 		// credential Envs are set globally.
 		globalIsEnvCreds = true
@@ -105,9 +197,9 @@ func handleCommonEnvVars() {
 	}
 
 	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
-		browserFlag, err := ParseBrowserFlag(browser)
+		browserFlag, err := ParseBoolFlag(browser)
 		if err != nil {
-			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Unable to validate MINIO_BROWSER environment variable")
+			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -123,9 +215,93 @@ func handleCommonEnvVars() {
 		logger.FatalIf(err, "error opening file %s", traceFile)
 	}
 
-	globalDomainName = os.Getenv("MINIO_DOMAIN")
-	if globalDomainName != "" {
-		globalIsEnvDomainName = true
+	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	if ok {
+		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
+
+		var etcdSecure bool
+		for _, endpoint := range etcdEndpoints {
+			u, err := xnet.ParseURL(endpoint)
+			if err != nil {
+				logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+			}
+			// If one of the endpoint is https, we will use https directly.
+			etcdSecure = etcdSecure || u.Scheme == "https"
+		}
+
+		var err error
+		if etcdSecure {
+			// This is only to support client side certificate authentication
+			// https://coreos.com/etcd/docs/latest/op-guide/security.html
+			etcdClientCertFile, ok1 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT")
+			etcdClientCertKey, ok2 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT_KEY")
+			var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+			if ok1 && ok2 {
+				getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+					cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
+					return &cert, terr
+				}
+			}
+
+			globalEtcdClient, err = etcd.New(etcd.Config{
+				Endpoints:         etcdEndpoints,
+				DialTimeout:       defaultDialTimeout,
+				DialKeepAliveTime: defaultDialKeepAlive,
+				TLS: &tls.Config{
+					RootCAs:              globalRootCAs,
+					GetClientCertificate: getClientCertificate,
+				},
+			})
+		} else {
+			globalEtcdClient, err = etcd.New(etcd.Config{
+				Endpoints:         etcdEndpoints,
+				DialTimeout:       defaultDialTimeout,
+				DialKeepAliveTime: defaultDialKeepAlive,
+			})
+		}
+		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+	}
+
+	v, ok := os.LookupEnv("MINIO_DOMAIN")
+	if ok {
+		for _, domainName := range strings.Split(v, ",") {
+			if _, ok = dns2.IsDomainName(domainName); !ok {
+				logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
+					"Invalid MINIO_DOMAIN value in environment variable")
+			}
+			globalDomainNames = append(globalDomainNames, domainName)
+		}
+	}
+
+	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if ok {
+		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
+		var domainIPs = set.NewStringSet()
+		for _, endpoint := range minioEndpoints {
+			if net.ParseIP(endpoint) == nil {
+				// Checking if the IP is a DNS entry.
+				addrs, err := net.LookupHost(endpoint)
+				if err != nil {
+					logger.FatalIf(err, "Unable to initialize Minio server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
+				}
+				for _, addr := range addrs {
+					domainIPs.Add(addr)
+				}
+				continue
+			}
+			domainIPs.Add(endpoint)
+		}
+		updateDomainIPs(domainIPs)
+	} else {
+		// Add found interfaces IP address to global domain IPS,
+		// loopback addresses will be naturally dropped.
+		updateDomainIPs(localIP4)
+	}
+
+	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+		var err error
+		globalDNSConfig, err = dns.NewCoreDNS(globalDomainNames, globalDomainIPs, globalMinioPort, globalEtcdClient)
+		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainNames)
 	}
 
 	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
@@ -153,6 +329,16 @@ func handleCommonEnvVars() {
 		globalCacheExpiry = expiry
 	}
 
+	if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" {
+		maxUse, err := strconv.Atoi(maxUseStr)
+		if err != nil {
+			logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr)
+		}
+		// maxUse should be a valid percentage.
+		if maxUse > 0 && maxUse <= 100 {
+			globalCacheMaxUse = maxUse
+		}
+	}
 	// In place update is true by default if the MINIO_UPDATE is not set
 	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
 	// in-place update is off.
@@ -189,5 +375,39 @@ func handleCommonEnvVars() {
 	}
 
 	// Get WORM environment variable.
-	globalWORMEnabled = strings.EqualFold(os.Getenv("MINIO_WORM"), "on")
+	if worm := os.Getenv("MINIO_WORM"); worm != "" {
+		wormFlag, err := ParseBoolFlag(worm)
+		if err != nil {
+			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
+		}
+
+		// worm Envs are set globally, this does not represent
+		// if worm is turned off or on.
+		globalIsEnvWORM = true
+		globalWORMEnabled = bool(wormFlag)
+	}
+
+	if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
+		globalIsCompressionEnabled = strings.EqualFold(compress, "true")
+	}
+
+	compressExtensions := os.Getenv("MINIO_COMPRESS_EXTENSIONS")
+	compressMimeTypes := os.Getenv("MINIO_COMPRESS_MIMETYPES")
+	if compressExtensions != "" || compressMimeTypes != "" {
+		globalIsEnvCompression = true
+		if compressExtensions != "" {
+			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter))
+			if err != nil {
+				logger.Fatal(err, "Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", extensions)
+			}
+			globalCompressExtensions = extensions
+		}
+		if compressMimeTypes != "" {
+			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter))
+			if err != nil {
+				logger.Fatal(err, "Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", contenttypes)
+			}
+			globalCompressMimeTypes = contenttypes
+		}
+	}
 }

cmd/config-common.go

@@ -0,0 +1,151 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+)
+
+var errConfigNotFound = errors.New("config file not found")
+
+func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
+	var buffer bytes.Buffer
+	// Read entire content by setting size to -1
+	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
+		// Treat object not found as config not found.
+		if isErrObjectNotFound(err) {
+			return nil, errConfigNotFound
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
+		logger.LogIf(ctx, err)
+		return nil, err
+	}
+
+	// Return config not found on empty content.
+	if buffer.Len() == 0 {
+		return nil, errConfigNotFound
+	}
+
+	return buffer.Bytes(), nil
+}
+
+func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	_, err := client.Delete(ctx, configFile)
+	return err
+}
+
+func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+}
+
+func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	_, err := client.Put(timeoutCtx, configFile, string(data))
+	if err == context.DeadlineExceeded {
+		return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
+	} else if err != nil {
+		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
+	}
+	return nil
+}
+
+func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
+	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
+	if err != nil {
+		return err
+	}
+
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), ObjectOptions{})
+	return err
+}
+
+func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) ([]byte, error) {
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	resp, err := client.Get(timeoutCtx, configFile)
+	if err != nil {
+		if err == context.DeadlineExceeded {
+			return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
+		}
+		return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
+	}
+	if resp.Count == 0 {
+		return nil, errConfigNotFound
+	}
+	for _, ev := range resp.Kvs {
+		if string(ev.Key) == configFile {
+			return ev.Value, nil
+		}
+	}
+	return nil, errConfigNotFound
+}
+
+// watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
+func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	defer cancel()
+	for watchResp := range globalEtcdClient.Watch(ctx, configFile) {
+		for _, event := range watchResp.Events {
+			if event.IsModify() || event.IsCreate() {
+				loadCfgFn(objAPI)
+			}
+		}
+	}
+}
+
+func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	resp, err := client.Get(timeoutCtx, configFile)
+	if err != nil {
+		if err == context.DeadlineExceeded {
+			return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
+		}
+		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
+	}
+	if resp.Count == 0 {
+		return errConfigNotFound
+	}
+	return nil
+}
+
+func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	if globalEtcdClient != nil {
+		return checkConfigEtcd(ctx, globalEtcdClient, configFile)
+	}
+
+	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
+		// Treat object not found as config not found.
+		if isErrObjectNotFound(err) {
+			return errConfigNotFound
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
+		logger.LogIf(ctx, err)
+		return err
+	}
+	return nil
+}

cmd/config-current.go

@@ -17,17 +17,22 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"os"
 	"reflect"
 	"sync"
 
+	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
-	"github.com/minio/minio/pkg/quick"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Steps to move from version N to version N+1
@@ -39,9 +44,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "23"
+const serverConfigVersion = "33"
 
-type serverConfig = serverConfigV23
+type serverConfig = serverConfigV33
 
 var (
 	// globalServerConfig server config.
@@ -62,11 +67,21 @@ func (s *serverConfig) SetRegion(region string) {
 
 // GetRegion get current region.
 func (s *serverConfig) GetRegion() string {
+	if globalIsEnvRegion {
+		return globalServerRegion
+	}
+	if s == nil {
+		return ""
+	}
 	return s.Region
 }
 
 // SetCredential sets new credential and returns the previous credential.
 func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
+	if creds.IsValid() && globalActiveCred.IsValid() {
+		globalActiveCred = creds
+	}
+
 	// Save previous credential.
 	prevCred = s.Credential
 
@@ -79,13 +94,16 @@ func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Cred
 
 // GetCredentials get current credentials.
 func (s *serverConfig) GetCredential() auth.Credentials {
+	if globalActiveCred.IsValid() {
+		return globalActiveCred
+	}
 	return s.Credential
 }
 
-// SetBrowser set if browser is enabled.
-func (s *serverConfig) SetBrowser(b bool) {
+// SetWorm set if worm is enabled.
+func (s *serverConfig) SetWorm(b bool) {
 	// Set the new value.
-	s.Browser = BrowserFlag(b)
+	s.Worm = BoolFlag(b)
 }
 
 func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
@@ -96,30 +114,292 @@ func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
 // GetStorageClass reads storage class fields from current config.
 // It returns the standard and reduced redundancy storage class struct
 func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
+	if globalIsStorageClass {
+		return globalStandardStorageClass, globalRRStorageClass
+	}
+	if s == nil {
+		return storageClass{}, storageClass{}
+	}
 	return s.StorageClass.Standard, s.StorageClass.RRS
 }
 
-// GetCredentials get current credentials.
-func (s *serverConfig) GetBrowser() bool {
-	return bool(s.Browser)
+// GetWorm get current credentials.
+func (s *serverConfig) GetWorm() bool {
+	if globalIsEnvWORM {
+		return globalWORMEnabled
+	}
+	if s == nil {
+		return false
+	}
+	return bool(s.Worm)
 }
 
 // SetCacheConfig sets the current cache config
-func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int) {
+func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxuse int) {
 	s.Cache.Drives = drives
 	s.Cache.Exclude = exclude
 	s.Cache.Expiry = expiry
+	s.Cache.MaxUse = maxuse
 }
 
 // GetCacheConfig gets the current cache config
 func (s *serverConfig) GetCacheConfig() CacheConfig {
+	if globalIsDiskCacheEnabled {
+		return CacheConfig{
+			Drives:  globalCacheDrives,
+			Exclude: globalCacheExcludes,
+			Expiry:  globalCacheExpiry,
+			MaxUse:  globalCacheMaxUse,
+		}
+	}
+	if s == nil {
+		return CacheConfig{}
+	}
 	return s.Cache
 }
 
-// Save config.
-func (s *serverConfig) Save() error {
-	// Save config file.
-	return quick.Save(getConfigFile(), s)
+func (s *serverConfig) Validate() error {
+	if s == nil {
+		return nil
+	}
+	if s.Version != serverConfigVersion {
+		return fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, s.Version)
+	}
+
+	// Validate credential fields only when
+	// they are not set via the environment
+	// Error out if global is env credential is not set and config has invalid credential
+	if !globalIsEnvCreds && !s.Credential.IsValid() {
+		return errors.New("invalid credential in config file")
+	}
+
+	// Region: nothing to validate
+	// Worm, Cache and StorageClass values are already validated during json unmarshal
+	for _, v := range s.Notify.AMQP {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("amqp: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.Elasticsearch {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("elasticsearch: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.Kafka {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("kafka: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.MQTT {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("mqtt: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.MySQL {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("mysql: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.NATS {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("nats: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.NSQ {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("nsq: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.PostgreSQL {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("postgreSQL: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.Redis {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("redis: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.Webhook {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("webhook: %s", err)
+		}
+	}
+
+	return nil
+}
+
+// SetCompressionConfig sets the current compression config
+func (s *serverConfig) SetCompressionConfig(extensions []string, mimeTypes []string) {
+	s.Compression.Extensions = extensions
+	s.Compression.MimeTypes = mimeTypes
+	s.Compression.Enabled = globalIsCompressionEnabled
+}
+
+// GetCompressionConfig gets the current compression config
+func (s *serverConfig) GetCompressionConfig() compressionConfig {
+	return s.Compression
+}
+
+func (s *serverConfig) loadFromEnvs() {
+	// If env is set override the credentials from config file.
+	if globalIsEnvCreds {
+		s.SetCredential(globalActiveCred)
+	}
+
+	if globalIsEnvWORM {
+		s.SetWorm(globalWORMEnabled)
+	}
+
+	if globalIsEnvRegion {
+		s.SetRegion(globalServerRegion)
+	}
+
+	if globalIsStorageClass {
+		s.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
+	}
+
+	if globalIsDiskCacheEnabled {
+		s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
+	}
+
+	if err := Environment.LookupKMSConfig(s.KMS); err != nil {
+		logger.FatalIf(err, "Unable to setup the KMS")
+	}
+
+	if globalIsEnvCompression {
+		s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
+	}
+
+	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
+		if u, err := xnet.ParseURL(jwksURL); err == nil {
+			s.OpenID.JWKS.URL = u
+			logger.FatalIf(s.OpenID.JWKS.PopulatePublicKey(), "Unable to populate public key from JWKS URL")
+		}
+	}
+
+	if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
+		if u, err := xnet.ParseURL(opaURL); err == nil {
+			s.Policy.OPA.URL = u
+			s.Policy.OPA.AuthToken = os.Getenv("MINIO_IAM_OPA_AUTHTOKEN")
+		}
+	}
+}
+
+// TestNotificationTargets tries to establish connections to all notification
+// targets when enabled. This is a good way to make sure all configurations
+// set by the user can work.
+func (s *serverConfig) TestNotificationTargets() error {
+	for k, v := range s.Notify.AMQP {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewAMQPTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("amqp(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Elasticsearch {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewElasticsearchTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("elasticsearch(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Kafka {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewKafkaTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("kafka(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.MQTT {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewMQTTTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("mqtt(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.MySQL {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewMySQLTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("mysql(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.NATS {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewNATSTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("nats(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.NSQ {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewNSQTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("nsq(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.PostgreSQL {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewPostgreSQLTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("postgreSQL(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Redis {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewRedisTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("redis(%s): %s", k, err.Error())
+		}
+		t.Close()
+
+	}
+
+	return nil
 }
 
 // Returns the string describing a difference with the given
@@ -133,18 +413,18 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "Credential configuration differs"
 	case s.Region != t.Region:
 		return "Region configuration differs"
-	case s.Browser != t.Browser:
-		return "Browser configuration differs"
-	case s.Domain != t.Domain:
-		return "Domain configuration differs"
 	case s.StorageClass != t.StorageClass:
 		return "StorageClass configuration differs"
 	case !reflect.DeepEqual(s.Cache, t.Cache):
 		return "Cache configuration differs"
+	case !reflect.DeepEqual(s.Compression, t.Compression):
+		return "Compression configuration differs"
 	case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
 		return "AMQP Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
 		return "NATS Notification configuration differs"
+	case !reflect.DeepEqual(s.Notify.NSQ, t.Notify.NSQ):
+		return "NSQ Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.Elasticsearch, t.Notify.Elasticsearch):
 		return "ElasticSearch Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.Redis, t.Notify.Redis):
@@ -159,6 +439,10 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "MySQL Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.MQTT, t.Notify.MQTT):
 		return "MQTT Notification configuration differs"
+	case !reflect.DeepEqual(s.Logger, t.Logger):
+		return "Logger configuration differs"
+	case !reflect.DeepEqual(s.KMS, t.KMS):
+		return "KMS configuration differs"
 	case reflect.DeepEqual(s, t):
 		return ""
 	default:
@@ -176,7 +460,6 @@ func newServerConfig() *serverConfig {
 		Version:    serverConfigVersion,
 		Credential: cred,
 		Region:     globalMinioDefaultRegion,
-		Browser:    true,
 		StorageClass: storageClassConfig{
 			Standard: storageClass{},
 			RRS:      storageClass{},
@@ -185,8 +468,15 @@ func newServerConfig() *serverConfig {
 			Drives:  []string{},
 			Exclude: []string{},
 			Expiry:  globalCacheExpiry,
+			MaxUse:  globalCacheMaxUse,
 		},
+		KMS:    crypto.KMSConfig{},
 		Notify: notifier{},
+		Compression: compressionConfig{
+			Enabled:    false,
+			Extensions: globalCompressExtensions,
+			MimeTypes:  globalCompressMimeTypes,
+		},
 	}
 
 	// Make sure to initialize notification configs.
@@ -200,6 +490,8 @@ func newServerConfig() *serverConfig {
 	srvCfg.Notify.Redis["1"] = target.RedisArgs{}
 	srvCfg.Notify.NATS = make(map[string]target.NATSArgs)
 	srvCfg.Notify.NATS["1"] = target.NATSArgs{}
+	srvCfg.Notify.NSQ = make(map[string]target.NSQArgs)
+	srvCfg.Notify.NSQ["1"] = target.NSQArgs{}
 	srvCfg.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
 	srvCfg.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{}
 	srvCfg.Notify.MySQL = make(map[string]target.MySQLArgs)
@@ -212,154 +504,147 @@ func newServerConfig() *serverConfig {
 	srvCfg.Cache.Drives = make([]string, 0)
 	srvCfg.Cache.Exclude = make([]string, 0)
 	srvCfg.Cache.Expiry = globalCacheExpiry
+	srvCfg.Cache.MaxUse = globalCacheMaxUse
+
+	// Console logging is on by default
+	srvCfg.Logger.Console.Enabled = true
+	// Create an example of HTTP logger
+	srvCfg.Logger.HTTP = make(map[string]loggerHTTP)
+	srvCfg.Logger.HTTP["target1"] = loggerHTTP{Endpoint: "https://username:password@example.com/api"}
+
 	return srvCfg
 }
 
-// newConfig - initialize a new server config, saves env parameters if
+func (s *serverConfig) loadToCachedConfigs() {
+	if !globalIsEnvCreds {
+		globalActiveCred = s.GetCredential()
+	}
+	if !globalIsEnvWORM {
+		globalWORMEnabled = s.GetWorm()
+	}
+	if !globalIsEnvRegion {
+		globalServerRegion = s.GetRegion()
+	}
+	if !globalIsStorageClass {
+		globalStandardStorageClass, globalRRStorageClass = s.GetStorageClass()
+	}
+	if !globalIsDiskCacheEnabled {
+		cacheConf := s.GetCacheConfig()
+		globalCacheDrives = cacheConf.Drives
+		globalCacheExcludes = cacheConf.Exclude
+		globalCacheExpiry = cacheConf.Expiry
+		globalCacheMaxUse = cacheConf.MaxUse
+	}
+	if err := Environment.LookupKMSConfig(s.KMS); err != nil {
+		logger.FatalIf(err, "Unable to setup the KMS")
+	}
+
+	if !globalIsCompressionEnabled {
+		compressionConf := s.GetCompressionConfig()
+		globalCompressExtensions = compressionConf.Extensions
+		globalCompressMimeTypes = compressionConf.MimeTypes
+		globalIsCompressionEnabled = compressionConf.Enabled
+	}
+
+	globalIAMValidators = getAuthValidators(s)
+
+	if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
+		globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
+			URL:         s.Policy.OPA.URL,
+			AuthToken:   s.Policy.OPA.AuthToken,
+			Transport:   NewCustomHTTPTransport(),
+			CloseRespFn: xhttp.DrainBody,
+		})
+	}
+}
+
+// newSrvConfig - initialize a new server config, saves env parameters if
 // found, otherwise use default parameters
-func newConfig() error {
+func newSrvConfig(objAPI ObjectLayer) error {
 	// Initialize server config.
 	srvCfg := newServerConfig()
 
-	// If env is set override the credentials from config file.
-	if globalIsEnvCreds {
-		srvCfg.SetCredential(globalActiveCred)
-	}
+	// Override any values from ENVs.
+	srvCfg.loadFromEnvs()
 
-	if globalIsEnvBrowser {
-		srvCfg.SetBrowser(globalIsBrowserEnabled)
-	}
-
-	if globalIsEnvRegion {
-		srvCfg.SetRegion(globalServerRegion)
-	}
-
-	if globalIsEnvDomainName {
-		srvCfg.Domain = globalDomainName
-	}
-
-	if globalIsStorageClass {
-		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
-	}
-
-	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
-	}
+	// Load values to cached global values.
+	srvCfg.loadToCachedConfigs()
 
 	// hold the mutex lock before a new config is assigned.
-	// Save the new config globally.
-	// unlock the mutex.
 	globalServerConfigMu.Lock()
 	globalServerConfig = srvCfg
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return globalServerConfig.Save()
+	return saveServerConfig(context.Background(), objAPI, globalServerConfig)
 }
 
 // getValidConfig - returns valid server configuration
-func getValidConfig() (*serverConfig, error) {
-	srvCfg := &serverConfig{
-		Region:  globalMinioDefaultRegion,
-		Browser: true,
-	}
-
-	if _, err := quick.Load(getConfigFile(), srvCfg); err != nil {
+func getValidConfig(objAPI ObjectLayer) (*serverConfig, error) {
+	srvCfg, err := readServerConfig(context.Background(), objAPI)
+	if err != nil {
 		return nil, err
 	}
 
-	if srvCfg.Version != serverConfigVersion {
-		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
-	}
-
-	// Validate credential fields only when
-	// they are not set via the environment
-	// Error out if global is env credential is not set and config has invalid credential
-	if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
-		return nil, errors.New("invalid credential in config file " + getConfigFile())
-	}
-
-	return srvCfg, nil
+	return srvCfg, srvCfg.Validate()
 }
 
 // loadConfig - loads a new config from disk, overrides params from env
 // if found and valid
-func loadConfig() error {
-	srvCfg, err := getValidConfig()
+func loadConfig(objAPI ObjectLayer) error {
+	srvCfg, err := getValidConfig(objAPI)
 	if err != nil {
 		return uiErrInvalidConfig(nil).Msg(err.Error())
 	}
 
-	// If env is set override the credentials from config file.
-	if globalIsEnvCreds {
-		srvCfg.SetCredential(globalActiveCred)
-	}
+	// Override any values from ENVs.
+	srvCfg.loadFromEnvs()
 
-	if globalIsEnvBrowser {
-		srvCfg.SetBrowser(globalIsBrowserEnabled)
-	}
-
-	if globalIsEnvRegion {
-		srvCfg.SetRegion(globalServerRegion)
-	}
-
-	if globalIsEnvDomainName {
-		srvCfg.Domain = globalDomainName
-	}
-
-	if globalIsStorageClass {
-		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
-	}
-
-	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
-	}
+	// Load values to cached global values.
+	srvCfg.loadToCachedConfigs()
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()
 	globalServerConfig = srvCfg
-	if !globalIsEnvCreds {
-		globalActiveCred = globalServerConfig.GetCredential()
-	}
-	if !globalIsEnvBrowser {
-		globalIsBrowserEnabled = globalServerConfig.GetBrowser()
-	}
-	if !globalIsEnvRegion {
-		globalServerRegion = globalServerConfig.GetRegion()
-	}
-	if !globalIsEnvDomainName {
-		globalDomainName = globalServerConfig.Domain
-	}
-	if !globalIsStorageClass {
-		globalStandardStorageClass, globalRRStorageClass = globalServerConfig.GetStorageClass()
-	}
-	if !globalIsDiskCacheEnabled {
-		cacheConf := globalServerConfig.GetCacheConfig()
-		globalCacheDrives = cacheConf.Drives
-		globalCacheExcludes = cacheConf.Exclude
-		globalCacheExpiry = cacheConf.Expiry
-	}
 	globalServerConfigMu.Unlock()
 
 	return nil
 }
 
+// getAuthValidators - returns ValidatorList which contains
+// enabled providers in server config.
+// A new authentication provider is added like below
+// * Add a new provider in pkg/iam/validator package.
+func getAuthValidators(config *serverConfig) *validator.Validators {
+	validators := validator.NewValidators()
+
+	if config.OpenID.JWKS.URL != nil {
+		validators.Add(validator.NewJWT(config.OpenID.JWKS))
+	}
+
+	return validators
+}
+
 // getNotificationTargets - returns TargetList which contains enabled targets in serverConfig.
 // A new notification target is added like below
 // * Add a new target in pkg/event/target package.
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
-func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
+func getNotificationTargets(config *serverConfig) *event.TargetList {
 	targetList := event.NewTargetList()
-
+	if config == nil {
+		return targetList
+	}
 	for id, args := range config.Notify.AMQP {
 		if args.Enable {
 			newTarget, err := target.NewAMQPTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -368,10 +653,14 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewElasticsearchTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+
 			}
 		}
 	}
@@ -380,22 +669,27 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewKafkaTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
 
 	for id, args := range config.Notify.MQTT {
 		if args.Enable {
+			args.RootCAs = globalRootCAs
 			newTarget, err := target.NewMQTTTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -404,10 +698,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewMySQLTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -416,10 +712,26 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewNATSTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+			}
+		}
+	}
+
+	for id, args := range config.Notify.NSQ {
+		if args.Enable {
+			newTarget, err := target.NewNSQTarget(id, args)
+			if err != nil {
+				logger.LogIf(context.Background(), err)
+				continue
+			}
+			if err = targetList.Add(newTarget); err != nil {
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -428,10 +740,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewPostgreSQLTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -440,22 +754,26 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewRedisTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
 
 	for id, args := range config.Notify.Webhook {
 		if args.Enable {
+			args.RootCAs = globalRootCAs
 			newTarget := target.NewWebhookTarget(id, args)
 			if err := targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
 
-	return targetList, nil
+	return targetList
 }

cmd/config-current_test.go

@@ -17,9 +17,9 @@
 package cmd
 
 import (
-	"io/ioutil"
+	"context"
 	"os"
-	"path/filepath"
+	"path"
 	"testing"
 
 	"github.com/minio/minio/pkg/auth"
@@ -27,12 +27,15 @@ import (
 )
 
 func TestServerConfig(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	objLayer, fsDir, err := prepareFS()
 	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
 		t.Fatalf("Init Test config failed")
 	}
-	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
 
 	if globalServerConfig.GetRegion() != globalMinioDefaultRegion {
 		t.Errorf("Expecting region `us-east-1` found %s", globalServerConfig.GetRegion())
@@ -49,16 +52,12 @@ func TestServerConfig(t *testing.T) {
 		t.Errorf("Expecting version %s found %s", globalServerConfig.GetVersion(), serverConfigVersion)
 	}
 
-	// Attempt to save.
-	if err := globalServerConfig.Save(); err != nil {
+	if err := saveServerConfig(context.Background(), objLayer, globalServerConfig); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 
-	// Do this only once here.
-	setConfigDir(rootPath)
-
 	// Initialize server config.
-	if err := loadConfig(); err != nil {
+	if err := loadConfig(objLayer); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 }
@@ -68,6 +67,9 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	os.Setenv("MINIO_BROWSER", "off")
 	defer os.Unsetenv("MINIO_BROWSER")
 
+	os.Setenv("MINIO_WORM", "on")
+	defer os.Unsetenv("MINIO_WORM")
+
 	os.Setenv("MINIO_ACCESS_KEY", "minio")
 	defer os.Unsetenv("MINIO_ACCESS_KEY")
 
@@ -82,61 +84,70 @@ func TestServerConfigWithEnvs(t *testing.T) {
 
 	defer resetGlobalIsEnvs()
 
-	// Get test root.
-	rootPath, err := getTestRoot()
+	objLayer, fsDir, err := prepareFS()
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
+	defer os.RemoveAll(fsDir)
+
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = objLayer
+	globalObjLayerMutex.Unlock()
 
 	serverHandleEnvVars()
 
-	// Do this only once here.
-	setConfigDir(rootPath)
-
 	// Init config
-	initConfig()
+	initConfig(objLayer)
 
-	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
-
-	// Check if serverConfig has
-	if globalServerConfig.GetBrowser() {
-		t.Errorf("Expecting browser is set to false found %v", globalServerConfig.GetBrowser())
+	// Check if serverConfig has browser disabled
+	if globalIsBrowserEnabled {
+		t.Error("Expected browser to be disabled but it is not")
 	}
 
-	// Check if serverConfig has
+	// Check if serverConfig returns WORM config from the env
+	if !globalServerConfig.GetWorm() {
+		t.Error("Expected WORM to be enabled but it is not")
+	}
+
+	// Check if serverConfig has region from the environment
 	if globalServerConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expecting region to be \"us-west-1\" found %v", globalServerConfig.GetRegion())
+		t.Errorf("Expected region to be \"us-west-1\", found %v", globalServerConfig.GetRegion())
 	}
 
-	// Check if serverConfig has
+	// Check if serverConfig has credentials from the environment
 	cred := globalServerConfig.GetCredential()
 
 	if cred.AccessKey != "minio" {
-		t.Errorf("Expecting access key to be `minio` found %s", cred.AccessKey)
+		t.Errorf("Expected access key to be `minio`, found %s", cred.AccessKey)
 	}
 
 	if cred.SecretKey != "minio123" {
-		t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
+		t.Errorf("Expected access key to be `minio123`, found %s", cred.SecretKey)
 	}
 
-	if globalServerConfig.Domain != "domain.com" {
-		t.Errorf("Expecting Domain to be `domain.com` found " + globalServerConfig.Domain)
+	// Check if serverConfig has the correct domain
+	if globalDomainNames[0] != "domain.com" {
+		t.Errorf("Expected Domain to be `domain.com`, found " + globalDomainNames[0])
 	}
 }
 
 // Tests config validator..
 func TestValidateConfig(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	objLayer, fsDir, err := prepareFS()
 	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
 		t.Fatalf("Init Test config failed")
 	}
 
-	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
-
-	configPath := filepath.Join(rootPath, minioConfigFile)
-
+	configPath := path.Join(minioConfigPrefix, minioConfigFile)
 	v := serverConfigVersion
 
 	testCases := []struct {
@@ -174,66 +185,69 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
 
 		// Test 11 - Test AMQP
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
 
 		// Test 12 - Test NATS
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
 
 		// Test 13 - Test ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
 
 		// Test 14 - Test Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
 
 		// Test 15 - Test PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
 		// Test 16 - Test Kafka
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
 
 		// Test 17 - Test Webhook
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
 
 		// Test 18 - Test MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
 		// Test 19 - Test Format for MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
 		// Test 20 - Test valid Format for MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
 		// Test 21 - Test Format for PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
 		// Test 22 - Test valid Format for PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
 		// Test 23 - Test Format for ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
 
 		// Test 24 - Test valid Format for ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
 
 		// Test 25 - Test Format for Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
 
 		// Test 26 - Test valid Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
 
 		// Test 27 - Test MQTT
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "username": "", "password": "", "queueDir": "", "queueLimit": 0}}}}`, false},
+
+		// Test 28 - Test NSQ
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": ""} }}}`, false},
 	}
 
 	for i, testCase := range testCases {
-		if werr := ioutil.WriteFile(configPath, []byte(testCase.configData), 0700); werr != nil {
-			t.Fatal(werr)
+		if err = saveConfig(context.Background(), objLayer, configPath, []byte(testCase.configData)); err != nil {
+			t.Fatal(err)
 		}
-		_, verr := getValidConfig()
-		if testCase.shouldPass && verr != nil {
-			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, verr)
+		_, err = getValidConfig(objLayer)
+		if testCase.shouldPass && err != nil {
+			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, err)
 		}
-		if !testCase.shouldPass && verr == nil {
+		if !testCase.shouldPass && err == nil {
 			t.Errorf("Test %d, should fail but it succeeded.", i+1)
 		}
 	}
@@ -249,76 +263,98 @@ func TestConfigDiff(t *testing.T) {
 		{&serverConfig{}, nil, "Given configuration is empty"},
 		// 2
 		{
-			&serverConfig{Credential: auth.Credentials{"u1", "p1"}},
-			&serverConfig{Credential: auth.Credentials{"u1", "p2"}},
+			&serverConfig{Credential: auth.Credentials{
+				AccessKey:  "u1",
+				SecretKey:  "p1",
+				Expiration: timeSentinel,
+			}},
+			&serverConfig{Credential: auth.Credentials{
+				AccessKey:  "u1",
+				SecretKey:  "p2",
+				Expiration: timeSentinel,
+			}},
 			"Credential configuration differs",
 		},
 		// 3
 		{&serverConfig{Region: "us-east-1"}, &serverConfig{Region: "us-west-1"}, "Region configuration differs"},
 		// 4
-		{&serverConfig{Browser: false}, &serverConfig{Browser: true}, "Browser configuration differs"},
-		// 5
-		{&serverConfig{Domain: "domain1"}, &serverConfig{Domain: "domain2"}, "Domain configuration differs"},
-		// 6
 		{
 			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 6}}},
 			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 4}}},
 			"StorageClass configuration differs",
 		},
-		// 7
+		// 5
 		{
 			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: false}}}},
 			"AMQP Notification configuration differs",
 		},
-		// 8
+		// 6
 		{
 			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: false}}}},
 			"NATS Notification configuration differs",
 		},
-		// 9
+		// 7
+		{
+			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: true}}}},
+			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: false}}}},
+			"NSQ Notification configuration differs",
+		},
+		// 8
 		{
 			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: false}}}},
 			"ElasticSearch Notification configuration differs",
 		},
-		// 10
+		// 9
 		{
 			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: false}}}},
 			"Redis Notification configuration differs",
 		},
-		// 11
+		// 10
 		{
 			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: false}}}},
 			"PostgreSQL Notification configuration differs",
 		},
-		// 12
+		// 11
 		{
 			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: false}}}},
 			"Kafka Notification configuration differs",
 		},
-		// 13
+		// 12
 		{
 			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: false}}}},
 			"Webhook Notification configuration differs",
 		},
-		// 14
+		// 13
 		{
 			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: false}}}},
 			"MySQL Notification configuration differs",
 		},
-		// 15
+		// 14
 		{
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
 			"MQTT Notification configuration differs",
 		},
+		// 15
+		{
+			&serverConfig{Logger: loggerConfig{
+				Console: loggerConsole{Enabled: true},
+				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address1"}},
+			}},
+			&serverConfig{Logger: loggerConfig{
+				Console: loggerConsole{Enabled: true},
+				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address2"}},
+			}},
+			"Logger configuration differs",
+		},
 	}
 
 	for i, testCase := range testCases {

cmd/config-dir.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"os"
 	"path/filepath"
-	"sync"
 
 	homedir "github.com/mitchellh/go-homedir"
 )
@@ -28,9 +27,6 @@ const (
 	// Default minio configuration directory where below configuration files/directories are stored.
 	defaultMinioConfigDir = ".minio"
 
-	// Minio configuration file.
-	minioConfigFile = "config.json"
-
 	// Directory contains below files/directories for HTTPS configuration.
 	certsDir = "certs"
 
@@ -44,55 +40,9 @@ const (
 	privateKeyFile = "private.key"
 )
 
-// ConfigDir - configuration directory with locking.
+// ConfigDir - points to a user set directory.
 type ConfigDir struct {
-	sync.Mutex
-	dir string
-}
-
-// Set - saves given directory as configuration directory.
-func (config *ConfigDir) Set(dir string) {
-	config.Lock()
-	defer config.Unlock()
-
-	config.dir = dir
-}
-
-// Get - returns current configuration directory.
-func (config *ConfigDir) Get() string {
-	config.Lock()
-	defer config.Unlock()
-
-	return config.dir
-}
-
-func (config *ConfigDir) getCertsDir() string {
-	return filepath.Join(config.Get(), certsDir)
-}
-
-// GetCADir - returns certificate CA directory.
-func (config *ConfigDir) GetCADir() string {
-	return filepath.Join(config.getCertsDir(), certsCADir)
-}
-
-// Create - creates configuration directory tree.
-func (config *ConfigDir) Create() error {
-	return os.MkdirAll(config.GetCADir(), 0700)
-}
-
-// GetMinioConfigFile - returns absolute path of config.json file.
-func (config *ConfigDir) GetMinioConfigFile() string {
-	return filepath.Join(config.Get(), minioConfigFile)
-}
-
-// GetPublicCertFile - returns absolute path of public.crt file.
-func (config *ConfigDir) GetPublicCertFile() string {
-	return filepath.Join(config.getCertsDir(), publicCertFile)
-}
-
-// GetPrivateKeyFile - returns absolute path of private.key file.
-func (config *ConfigDir) GetPrivateKeyFile() string {
-	return filepath.Join(config.getCertsDir(), privateKeyFile)
+	path string
 }
 
 func getDefaultConfigDir() string {
@@ -104,32 +54,54 @@ func getDefaultConfigDir() string {
 	return filepath.Join(homeDir, defaultMinioConfigDir)
 }
 
-var configDir = &ConfigDir{dir: getDefaultConfigDir()}
-
-func setConfigDir(dir string) {
-	configDir.Set(dir)
+func getDefaultCertsDir() string {
+	return filepath.Join(getDefaultConfigDir(), certsDir)
 }
 
-func getConfigDir() string {
-	return configDir.Get()
+func getDefaultCertsCADir() string {
+	return filepath.Join(getDefaultCertsDir(), certsCADir)
 }
 
-func getCADir() string {
-	return configDir.GetCADir()
+var (
+	// Default config, certs and CA directories.
+	defaultConfigDir  = &ConfigDir{path: getDefaultConfigDir()}
+	defaultCertsDir   = &ConfigDir{path: getDefaultCertsDir()}
+	defaultCertsCADir = &ConfigDir{path: getDefaultCertsCADir()}
+
+	// Points to current configuration directory -- deprecated, to be removed in future.
+	globalConfigDir = defaultConfigDir
+	// Points to current certs directory set by user with --certs-dir
+	globalCertsDir = defaultCertsDir
+	// Points to relative path to certs directory and is <value-of-certs-dir>/CAs
+	globalCertsCADir = defaultCertsCADir
+)
+
+// Get - returns current directory.
+func (dir *ConfigDir) Get() string {
+	return dir.path
 }
 
-func createConfigDir() error {
-	return configDir.Create()
+// Attempts to create all directories, ignores any permission denied errors.
+func mkdirAllIgnorePerm(path string) error {
+	err := os.MkdirAll(path, 0700)
+	if err != nil {
+		// It is possible in kubernetes like deployments this directory
+		// is already mounted and is not writable, ignore any write errors.
+		if os.IsPermission(err) {
+			err = nil
+		}
+	}
+	return err
 }
 
 func getConfigFile() string {
-	return configDir.GetMinioConfigFile()
+	return filepath.Join(globalConfigDir.Get(), minioConfigFile)
 }
 
 func getPublicCertFile() string {
-	return configDir.GetPublicCertFile()
+	return filepath.Join(globalCertsDir.Get(), publicCertFile)
 }
 
 func getPrivateKeyFile() string {
-	return configDir.GetPrivateKeyFile()
+	return filepath.Join(globalCertsDir.Get(), privateKeyFile)
 }