Ignore permission errors on config-dir (#6894)

Fixes #6873

.github/ISSUE_TEMPLATE.md

@@ -24,6 +24,10 @@
 <!--- How has this issue affected you? What are you trying to accomplish? -->
 <!--- Providing context helps us come up with a solution that is most useful in the real world -->
 
+## Regression
+<!-- Is this issue a regression? (Yes / No) -->
+<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
+
 ## Your Environment
 <!--- Include as many relevant details about the environment you experienced the bug in -->
 * Version used (`minio version`):

.github/PULL_REQUEST_TEMPLATE.md

@@ -7,6 +7,10 @@
 <!--- Why is this change required? What problem does it solve? -->
 <!--- If it fixes an open issue, please link to the issue here. -->
 
+## Regression
+<!-- Is this PR fixing a regression? (Yes / No) -->
+<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
+
 ## How Has This Been Tested?
 <!--- Please describe in detail how you tested your changes. -->
 <!--- Include details of your testing environment, and the tests you ran to -->

.gitignore

@@ -22,3 +22,4 @@ parts/
 prime/
 stage/
 .sia_temp/
+config.json

.travis.yml

@@ -1,34 +1,41 @@
 go_import_path: github.com/minio/minio
-sudo: required
-
-services:
- - docker
-
-dist: trusty
 
 language: go
 
-os:
-- linux
+# this ensures PRs based on a local branch are not built twice
+# the downside is that a PR targeting a different branch is not built
+# but as a workaround you can add the branch to this list
+branches:
+  only:
+    - master
 
-env:
-- ARCH=x86_64
+matrix:
+  include:
+    - os: linux
+      dist: trusty
+      sudo: required
+      env:
+        - ARCH=x86_64
+      go: 1.10.4
+      script:
+        - make
+        - diff -au <(gofmt -s -d cmd) <(printf "")
+        - diff -au <(gofmt -s -d pkg) <(printf "")
+        - make test GOFLAGS="-timeout 15m -race -v"
+        - make verify
+        - make coverage
+        - cd browser && yarn && yarn test && cd ..
+    - os: windows
+      env:
+        - ARCH=x86_64
+      go: 1.10.4
+      script:
+        - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
+        - for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
+        - bash buildscripts/go-coverage.sh
 
 before_install:
-- nvm install stable
-
-script:
-## Run all the tests
-- make
-- diff -au <(gofmt -s -d cmd) <(printf "")
-- diff -au <(gofmt -s -d pkg) <(printf "")
-- make test GOFLAGS="-timeout 15m -race -v"
-- make coverage
-- node --version
-- cd browser && yarn && yarn test && cd ..
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
 
 after_success:
 - bash <(curl -s https://codecov.io/bash)
-
-go:
-- '1.10.1'

Dockerfile

@@ -14,7 +14,7 @@ WORKDIR /go/src/github.com/minio/
 COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 
 RUN  \
-     apk add --no-cache ca-certificates curl && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      apk add --no-cache --virtual .build-deps git && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      go get -v -d github.com/minio/minio && \

Dockerfile.dev

@@ -1,27 +1,20 @@
-FROM golang:1.10.1-alpine3.7
+FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
-ENV GOPATH /go
-ENV PATH $PATH:$GOPATH/bin
-ENV CGO_ENABLED 0
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY minio /usr/bin/
+
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
-WORKDIR /go/src/github.com/minio/
-
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
-
-COPY . /go/src/github.com/minio/minio
-
-RUN  \
-     apk add --no-cache ca-certificates curl && \
-     apk add --no-cache --virtual .build-deps git && \
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck.sh
 
 EXPOSE 9000
 

Dockerfile.release

@@ -9,8 +9,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
 RUN \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps curl && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \

Makefile

@@ -13,7 +13,7 @@ checks:
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
 getdeps:
-	@echo "Installing golint" && go get -u github.com/golang/lint/golint
+	@echo "Installing golint" && go get -u golang.org/x/lint/golint
 	@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
 	@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
 	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
@@ -23,8 +23,8 @@ verifiers: getdeps vet fmt lint cyclo deadcode spelling
 
 vet:
 	@echo "Running $@"
-	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult cmd
-	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult pkg
+	@go tool vet cmd
+	@go tool vet pkg
 
 fmt:
 	@echo "Running $@"
@@ -42,23 +42,27 @@ ineffassign:
 
 cyclo:
 	@echo "Running $@"
-	@${GOPATH}/bin/gocyclo -over 100 cmd
-	@${GOPATH}/bin/gocyclo -over 100 pkg
+	@${GOPATH}/bin/gocyclo -over 200 cmd
+	@${GOPATH}/bin/gocyclo -over 200 pkg
 
 deadcode:
 	@echo "Running $@"
 	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
 
 spelling:
-	@${GOPATH}/bin/misspell -error `find cmd/`
-	@${GOPATH}/bin/misspell -error `find pkg/`
-	@${GOPATH}/bin/misspell -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find cmd/`
+	@${GOPATH}/bin/misspell -locale US -error `find pkg/`
+	@${GOPATH}/bin/misspell -locale US -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
+	@${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build
 	@echo "Running unit tests"
 	@go test $(GOFLAGS) -tags kqueue ./...
+
+verify: build
 	@echo "Verifying build"
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
@@ -71,6 +75,9 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
 
+docker: build
+	@docker build -t $(TAG) . -f Dockerfile.dev
+
 pkg-add:
 	@echo "Adding new package $(PKG)"
 	@${GOPATH}/bin/govendor add $(PKG)
@@ -89,7 +96,7 @@ pkg-list:
 # Builds minio and installs it to $GOPATH/bin.
 install: build
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
-	@cp $(PWD)/minio $(GOPATH)/bin/minio
+	@mkdir -p $(GOPATH)/bin && cp $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
 clean:

README.md

@@ -17,7 +17,7 @@ docker run -p 9000:9000 minio/minio server /data
 docker pull minio/minio:edge
 docker run -p 9000:9000 minio/minio:edge server /data
 ```
-Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
+Note: Docker will not display the autogenerated keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use autogenerated keys with containers. Please visit Minio Docker quickstart guide for more information [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
 ### Homebrew

appveyor.yml

@@ -1,55 +0,0 @@
-# version format
-version: "{build}"
-
-# Operating system (build VM template)
-os: Windows Server 2012 R2
-
-# Platform.
-platform: x64
-
-clone_folder: c:\gopath\src\github.com\minio\minio
-
-# Environment variables
-environment:
-  GOPATH: c:\gopath
-  GOROOT: c:\go
-
-# scripts that run after cloning repository
-install:
-  - set PATH=%GOPATH%\bin;%GOROOT%\bin;%PATH%
-  - go version
-  - go env
-  - python --version
-
-# To run your custom scripts instead of automatic MSBuild
-build_script:
-  # Compile
-  # We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
-  - ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
-  - appveyor AddCompilationMessage "Starting Compile"
-  - cd c:\gopath\src\github.com\minio\minio
-  - go run buildscripts/gen-ldflags.go > temp.txt
-  - set /p BUILD_LDFLAGS=<temp.txt
-  - go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
-  - appveyor AddCompilationMessage "Compile Success"
-
-# To run your custom scripts instead of automatic tests
-test_script:
-  # Unit tests
-  - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
-  - mkdir build\coverage
-  - for /f "" %%G in ('go list github.com/minio/minio/... ^| find /i /v "browser/"') do ( go test -v -timeout 20m -race %%G )
-  - go test -v -timeout 20m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
-  - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
-
-after_test:
-  - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
-  - ps: Push-AppveyorArtifact build\coverage\coverage.txt
-  - ps: Push-AppveyorArtifact build\coverage\coverage.html
-  # Upload coverage report.
-  - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
-  - pip install codecov
-  - codecov -X gcov -f "build\coverage\coverage.txt"
-
-# to disable deployment
-deploy: off

browser/app/js/alert/actions.js

@@ -22,7 +22,7 @@ export let alertId = 0
 export const set = alert => {
   const id = alertId++
   return (dispatch, getState) => {
-    if (alert.type !== "danger") {
+    if (alert.type !== "danger" || alert.autoClear) {
       setTimeout(() => {
         dispatch({
           type: CLEAR,

browser/app/js/browser/Login.js

@@ -25,14 +25,35 @@ import web from "../web"
 import { Redirect } from "react-router-dom"
 
 export class Login extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      accessKey: "",
+      secretKey: ""
+    }
+  }
+
+  // Handle field changes
+  accessKeyChange(e) {
+    this.setState({
+      accessKey: e.target.value
+    })
+  }
+
+  secretKeyChange(e) {
+    this.setState({
+      secretKey: e.target.value
+    })
+  }
+
   handleSubmit(event) {
     event.preventDefault()
     const { showAlert, history } = this.props
     let message = ""
-    if (!document.getElementById("accessKey").value) {
+    if (this.state.accessKey === "") {
       message = "Access Key cannot be empty"
     }
-    if (!document.getElementById("secretKey").value) {
+    if (this.state.secretKey === "") {
       message = "Secret Key cannot be empty"
     }
     if (message) {
@@ -41,8 +62,8 @@ export class Login extends React.Component {
     }
     web
       .Login({
-        username: document.getElementById("accessKey").value,
-        password: document.getElementById("secretKey").value
+        username: this.state.accessKey,
+        password: this.state.secretKey
       })
       .then(res => {
         history.push("/")
@@ -77,6 +98,8 @@ export class Login extends React.Component {
         <div className="l-wrap">
           <form onSubmit={this.handleSubmit.bind(this)}>
             <InputGroup
+              value={this.state.accessKey}
+              onChange={this.accessKeyChange.bind(this)}
               className="ig-dark"
               label="Access Key"
               id="accessKey"
@@ -87,6 +110,8 @@ export class Login extends React.Component {
               autoComplete="username"
             />
             <InputGroup
+              value={this.state.secretKey}
+              onChange={this.secretKeyChange.bind(this)}
               className="ig-dark"
               label="Secret Key"
               id="secretKey"

browser/app/js/browser/__tests__/Login.test.js

@@ -60,20 +60,25 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
     // case where both keys are empty - displays the second warning
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
 
     // case where access key is empty
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Access Key cannot be empty")
 
     // case where secret key is empty
-    document.getElementById("accessKey").value = "accessKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: ""
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
   })
@@ -85,11 +90,12 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
-    document.getElementById("accessKey").value = "accessKey"
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(web.Login).toHaveBeenCalledWith({
       "username": "accessKey", 

browser/app/js/buckets/BucketList.js

@@ -67,8 +67,12 @@ const mapDispatchToProps = dispatch => {
   return {
     fetchBuckets: () => dispatch(actionsBuckets.fetchBuckets()),
     setBucketList: buckets => dispatch(actionsBuckets.setList(buckets)),
-    selectBucket: bucket => dispatch(actionsBuckets.selectBucket(bucket))
+    selectBucket: (bucket, prefix) =>
+      dispatch(actionsBuckets.selectBucket(bucket, prefix))
   }
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(BucketList)
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(BucketList)

browser/app/js/buckets/Policy.js

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { READ_ONLY, WRITE_ONLY, READ_WRITE } from '../constants'
+import { READ_ONLY, WRITE_ONLY, READ_WRITE, NONE } from '../constants'
 
 import React from "react"
 import { connect } from "react-redux"
@@ -42,37 +42,40 @@ export class Policy extends React.Component {
   render() {
     const {policy, prefix} = this.props
     let newPrefix = prefix
-
     if (newPrefix === '')
       newPrefix = '*'
 
-    return (
-      <div className="pmb-list">
-        <div className="pmbl-item">
-          { newPrefix }
+    if (policy === NONE) {
+      return <noscript />
+    } else {
+      return (
+        <div className="pmb-list">
+          <div className="pmbl-item">
+            { newPrefix }
+          </div>
+          <div className="pmbl-item">
+            <select className="form-control"
+              disabled
+              value={ policy }>
+              <option value={ READ_ONLY }>
+                Read Only
+              </option>
+              <option value={ WRITE_ONLY }>
+                Write Only
+              </option>
+              <option value={ READ_WRITE }>
+                Read and Write
+              </option>
+            </select>
+          </div>
+          <div className="pmbl-item">
+            <button className="btn btn-block btn-danger" onClick={ this.removePolicy.bind(this) }>
+              Remove
+            </button>
+          </div>
         </div>
-        <div className="pmbl-item">
-          <select className="form-control"
-            disabled
-            value={ policy }>
-            <option value={ READ_ONLY }>
-              Read Only
-            </option>
-            <option value={ WRITE_ONLY }>
-              Write Only
-            </option>
-            <option value={ READ_WRITE }>
-              Read and Write
-            </option>
-          </select>
-        </div>
-        <div className="pmbl-item">
-          <button className="btn btn-block btn-danger" onClick={ this.removePolicy.bind(this) }>
-            Remove
-          </button>
-        </div>
-      </div>
-    )
+      )
+    }
   }
 }
 

browser/app/js/buckets/__tests__/Policy.test.js

@@ -17,7 +17,7 @@
 import React from "react"
 import { shallow, mount } from "enzyme"
 import { Policy } from "../Policy"
-import { READ_ONLY, WRITE_ONLY, READ_WRITE } from "../../constants"
+import { READ_ONLY, WRITE_ONLY, READ_WRITE, NONE } from "../../constants"
 import web from "../../web"
 
 jest.mock("../../web", () => ({
@@ -31,6 +31,11 @@ describe("Policy", () => {
     shallow(<Policy currentBucket={"bucket"} prefix={"foo"} policy={READ_ONLY} />)
   })
 
+  it("should not render when policy is listed as 'none'", () => {
+    const wrapper = shallow(<Policy currentBucket={"bucket"} prefix={"foo"} policy={NONE} />)
+    expect(wrapper.find(".pmb-list").length).toBe(0)
+  })
+
   it("should call web.setBucketPolicy and fetchPolicies on submit", () => {
     const fetchPolicies = jest.fn()
     const wrapper = shallow(

browser/app/js/constants.js

@@ -23,6 +23,7 @@ export const minioBrowserPrefix = p.slice(0, p.indexOf("/", 1))
 export const READ_ONLY = "readonly"
 export const WRITE_ONLY = "writeonly"
 export const READ_WRITE = "readwrite"
+export const NONE = "none"
 
 export const SHARE_OBJECT_EXPIRY_DAYS = 5
 export const SHARE_OBJECT_EXPIRY_HOURS = 0

browser/app/js/objects/ObjectActions.js

@@ -57,7 +57,7 @@ export class ObjectActions extends React.Component {
     })
   }
   render() {
-    const { object, showShareObjectModal } = this.props
+    const { object, showShareObjectModal, shareObjectName } = this.props
     return (
       <Dropdown id={`obj-actions-${object.name}`}>
         <Dropdown.Toggle noCaret className="fia-toggle" />
@@ -77,7 +77,8 @@ export class ObjectActions extends React.Component {
             <i className="fa fa-trash" />
           </a>
         </Dropdown.Menu>
-        {showShareObjectModal && <ShareObjectModal object={object} />}
+        {(showShareObjectModal && shareObjectName === object.name) &&
+          <ShareObjectModal object={object} />}
         {this.state.showDeleteConfirmation && (
           <DeleteObjectConfirmModal
             deleteObject={this.deleteObject.bind(this)}
@@ -92,7 +93,8 @@ export class ObjectActions extends React.Component {
 const mapStateToProps = (state, ownProps) => {
   return {
     object: ownProps.object,
-    showShareObjectModal: state.objects.shareObject.show
+    showShareObjectModal: state.objects.shareObject.show,
+    shareObjectName: state.objects.shareObject.object
   }
 }
 

browser/app/js/objects/ObjectItem.js

@@ -15,7 +15,6 @@
  */
 
 import React from "react"
-import classNames from "classnames"
 import { connect } from "react-redux"
 import humanize from "humanize"
 import Moment from "moment"
@@ -35,22 +34,19 @@ export const ObjectItem = ({
   onClick
 }) => {
   return (
-    <div 
-      className={
-        "fesl-row " + 
-        classNames({
-          "fesl-row-selected": checked
-        })
-      }
-      data-type={getDataType(name, contentType)}
-    >
+    <div className={"fesl-row"} data-type={getDataType(name, contentType)}>
       <div className="fesl-item fesl-item-icon">
         <div className="fi-select">
           <input
             type="checkbox"
             name={name}
+            checked={checked}
+            onChange={() => {
+              checked ? uncheckObject(name) : checkObject(name)
+            }}
           />
           <i className="fis-icon" />
+          <i className="fis-helper" />
         </div>
       </div>
       <div className="fesl-item fesl-item-name">
@@ -58,7 +54,9 @@ export const ObjectItem = ({
           href="#"
           onClick={e => {
             e.preventDefault()
-            checked ? uncheckObject(name) : checkObject(name)
+            if (onClick) {
+              onClick()
+            }
           }}
         >
           {name}

browser/app/js/objects/ObjectsBulkActions.js

@@ -29,10 +29,10 @@ export class ObjectsBulkActions extends React.Component {
     }
   }
   handleDownload() {
-    const { checkedObjects, resetCheckedList, downloadChecked, downloadObject } = this.props
-    if (checkedObjects.length === 1) {
+    const { checkedObjects, clearChecked, downloadChecked, downloadObject } = this.props
+    if (checkedObjects.length === 1 && !checkedObjects[0].endsWith('/')) {
       downloadObject(checkedObjects[0])
-      resetCheckedList()
+      clearChecked()
     } else {
       downloadChecked()
     }
@@ -59,17 +59,16 @@ export class ObjectsBulkActions extends React.Component {
         }
       >
         <span className="la-label">
-          <i className="fa fa-check-circle" /> {checkedObjects.length} 
+          <i className="fa fa-check-circle" /> {checkedObjects.length}
           {checkedObjects.length === 1 ? " Object " : " Objects "}
           selected
         </span>
         <span className="la-actions pull-right">
-          <button 
-            id="download-checked" 
-            onClick={this.handleDownload.bind(this)}
-          >
+          <button id="download-checked" onClick={this.handleDownload.bind(this)}>
             {" "}
-            Download {checkedObjects.length === 1 ? "object" : "all as zip"}{" "}
+            Download
+            {(checkedObjects.length === 1 && !checkedObjects[0].endsWith('/')) ? 
+            " object" : " all as zip" }{" "}
           </button>
         </span>
         <span className="la-actions pull-right">
@@ -105,6 +104,7 @@ const mapStateToProps = state => {
 
 const mapDispatchToProps = dispatch => {
   return {
+    downloadObject: object => dispatch(actions.downloadObject(object)),
     downloadChecked: () => dispatch(actions.downloadCheckedObjects()),
     downloadObject: object => dispatch(actions.downloadObject(object)),
     resetCheckedList: () => dispatch(actions.resetCheckedList()),

browser/app/js/objects/__tests__/ObjectActions.test.js

@@ -88,8 +88,21 @@ describe("ObjectActions", () => {
         object={{ name: "obj1" }}
         currentPrefix={"pre1/"}
         showShareObjectModal={true}
+        shareObjectName={"obj1"}
       />
     )
     expect(wrapper.find("Connect(ShareObjectModal)").length).toBe(1)
   })
+
+  it("shouldn't render ShareObjectModal when the names of the objects don't match", () => {
+    const wrapper = shallow(
+      <ObjectActions
+        object={{ name: "obj1" }}
+        currentPrefix={"pre1/"}
+        showShareObjectModal={true}
+        shareObjectName={"obj2"}
+      />
+    )
+    expect(wrapper.find("Connect(ShareObjectModal)").length).toBe(0)
+  })
 })

browser/app/js/objects/__tests__/ObjectItem.test.js

@@ -28,22 +28,40 @@ describe("ObjectItem", () => {
     expect(wrapper.prop("data-type")).toBe("image")
   })
 
-  it("should call checkObject when the object is selected", () => {
-    const checkObject = jest.fn()
-    const wrapper = shallow(<ObjectItem name={"test"} checked={false} checkObject={checkObject} />)
+  it("shouldn't call onClick when the object isclicked", () => {
+    const onClick = jest.fn()
+    const wrapper = shallow(<ObjectItem name={"test"} />)
     wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
+    expect(onClick).not.toHaveBeenCalled()
+  })
+
+  it("should call onClick when the folder isclicked", () => {
+    const onClick = jest.fn()
+    const wrapper = shallow(<ObjectItem name={"test/"} onClick={onClick} />)
+    wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
+    expect(onClick).toHaveBeenCalled()
+  })
+
+  it("should call checkObject when the object/prefix is checked", () => {
+    const checkObject = jest.fn()
+    const wrapper = shallow(
+      <ObjectItem name={"test"} checked={false} checkObject={checkObject} />
+    )
+    wrapper.find("input[type='checkbox']").simulate("change")
     expect(checkObject).toHaveBeenCalledWith("test")
   })
 
-  it("should render highlighted row when object is selected", () => {
+  it("should render checked checkbox", () => {
     const wrapper = shallow(<ObjectItem name={"test"} checked={true} />)
-    expect(wrapper.find(".fesl-row").hasClass("fesl-row-selected")).toBeTruthy()
+    expect(wrapper.find("input[type='checkbox']").prop("checked")).toBeTruthy()
   })
 
-  it("should call uncheckObject when the object is deselected", () => {
+  it("should call uncheckObject when the object/prefix is unchecked", () => {
     const uncheckObject = jest.fn()
-    const wrapper = shallow(<ObjectItem name={"test"} checked={true} uncheckObject={uncheckObject} />)
-    wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
+    const wrapper = shallow(
+      <ObjectItem name={"test"} checked={true} uncheckObject={uncheckObject} />
+    )
+    wrapper.find("input[type='checkbox']").simulate("change")
     expect(uncheckObject).toHaveBeenCalledWith("test")
   })
 })

browser/app/js/objects/__tests__/ObjectsBulkActions.test.js

@@ -20,7 +20,7 @@ import { ObjectsBulkActions } from "../ObjectsBulkActions"
 
 describe("ObjectsBulkActions", () => {
   it("should render without crashing", () => {
-    shallow(<ObjectsBulkActions checkedObjects={0} />)
+    shallow(<ObjectsBulkActions checkedObjects={[]} />)
   })
 
   it("should show actions when checkObjectsCount is more than 0", () => {
@@ -28,21 +28,33 @@ describe("ObjectsBulkActions", () => {
     expect(wrapper.hasClass("list-actions-toggled")).toBeTruthy()
   })
 
-  it("should call downloadObject for single object when download button is clicked", () => {
+  it("should call downloadObject when single object is selected and download button is clicked", () => {
     const downloadObject = jest.fn()
-    const resetCheckedList = jest.fn()
+    const clearChecked = jest.fn()
     const wrapper = shallow(
       <ObjectsBulkActions
-        checkedObjects={["test1"]}
+        checkedObjects={["test"]}
         downloadObject={downloadObject}
-        resetCheckedList={resetCheckedList}
+        clearChecked={clearChecked}
       />
     )
     wrapper.find("#download-checked").simulate("click")
     expect(downloadObject).toHaveBeenCalled()
   })
 
-  it("should call downloadChecked for multiple objects when download button is clicked", () => {
+  it("should call downloadChecked when a folder is selected and download button is clicked", () => {
+    const downloadChecked = jest.fn()
+    const wrapper = shallow(
+      <ObjectsBulkActions
+        checkedObjects={["test/"]}
+        downloadChecked={downloadChecked}
+      />
+    )
+    wrapper.find("#download-checked").simulate("click")
+    expect(downloadChecked).toHaveBeenCalled()
+  })
+
+  it("should call downloadChecked when multiple objects are selected and download button is clicked", () => {
     const downloadChecked = jest.fn()
     const wrapper = shallow(
       <ObjectsBulkActions

browser/app/js/objects/__tests__/actions.test.js

@@ -19,16 +19,29 @@ import thunk from "redux-thunk"
 import * as actionsObjects from "../actions"
 import * as alertActions from "../../alert/actions"
 import { minioBrowserPrefix } from "../../constants"
+import history from "../../history"
 
 jest.mock("../../web", () => ({
-  LoggedIn: jest.fn(() => true).mockReturnValueOnce(false),
-  ListObjects: jest.fn(() => {
-    return Promise.resolve({
-      objects: [{ name: "test1" }, { name: "test2" }],
-      istruncated: false,
-      nextmarker: "test2",
-      writable: false
-    })
+  LoggedIn: jest
+    .fn(() => true)
+    .mockReturnValueOnce(true)
+    .mockReturnValueOnce(false)
+    .mockReturnValueOnce(true)
+    .mockReturnValueOnce(true)
+    .mockReturnValueOnce(false),
+  ListObjects: jest.fn(({ bucketName }) => {
+    if (bucketName === "test-deny") {
+      return Promise.reject({
+        message: "listobjects is denied"
+      })
+    } else {
+      return Promise.resolve({
+        objects: [{ name: "test1" }, { name: "test2" }],
+        istruncated: false,
+        nextmarker: "test2",
+        writable: false
+      })
+    }
   }),
   RemoveObject: jest.fn(({ bucketName, objects }) => {
     if (!bucketName) {
@@ -160,6 +173,41 @@ describe("Objects actions", () => {
     })
   })
 
+  it("creates objects/RESET_LIST after failing to fetch the objects from bucket with ListObjects denied for LoggedIn users", () => {
+    const store = mockStore({
+      buckets: { currentBucket: "test-deny" },
+      objects: { currentPrefix: "" }
+    })
+    const expectedActions = [
+      {
+        type: "alert/SET",
+        alert: {
+          type: "danger",
+          message: "listobjects is denied",
+          id: alertActions.alertId,
+          autoClear: true
+        }
+      },
+      {
+        type: "object/RESET_LIST"
+      }
+    ]
+    return store.dispatch(actionsObjects.fetchObjects()).then(() => {
+      const actions = store.getActions()
+      expect(actions).toEqual(expectedActions)
+    })
+  })
+
+  it("redirect to login after failing to fetch the objects from bucket for non-LoggedIn users", () => {
+    const store = mockStore({
+      buckets: { currentBucket: "test-deny" },
+      objects: { currentPrefix: "" }
+    })
+    return store.dispatch(actionsObjects.fetchObjects()).then(() => {
+      expect(history.location.pathname.endsWith("/login")).toBeTruthy()
+    })
+  })
+
   it("creates objects/SET_SORT_BY and objects/SET_SORT_ORDER when sortObjects is called", () => {
     const store = mockStore({
       objects: {
@@ -244,7 +292,11 @@ describe("Objects actions", () => {
     const expectedActions = [
       {
         type: "alert/SET",
-        alert: { type: "danger", message: "Invalid bucket", id: 0 }
+        alert: {
+          type: "danger",
+          message: "Invalid bucket",
+          id: alertActions.alertId
+        }
       }
     ]
     return store.dispatch(actionsObjects.deleteObject("obj1")).then(() => {
@@ -355,7 +407,7 @@ describe("Objects actions", () => {
       store.dispatch(actionsObjects.downloadObject("obj1"))
       const url = `${
         window.location.origin
-      }${minioBrowserPrefix}/download/bk1/${encodeURI("pre1/obj1")}?token=''`
+      }${minioBrowserPrefix}/download/bk1/${encodeURI("pre1/obj1")}?token=`
       expect(setLocation).toHaveBeenCalledWith(url)
     })
 

browser/app/js/objects/actions.js

@@ -16,17 +16,15 @@
 
 import web from "../web"
 import history from "../history"
-import {
-  sortObjectsByName,
-  sortObjectsBySize,
-  sortObjectsByDate
-} from "../utils"
+import { sortObjectsByName, sortObjectsBySize, sortObjectsByDate } from "../utils"
 import { getCurrentBucket } from "../buckets/selectors"
 import { getCurrentPrefix, getCheckedList } from "./selectors"
 import * as alertActions from "../alert/actions"
+import * as bucketActions from "../buckets/actions"
 import { minioBrowserPrefix } from "../constants"
 
 export const SET_LIST = "objects/SET_LIST"
+export const RESET_LIST = "object/RESET_LIST"
 export const APPEND_LIST = "objects/APPEND_LIST"
 export const REMOVE = "objects/REMOVE"
 export const SET_SORT_BY = "objects/SET_SORT_BY"
@@ -45,6 +43,10 @@ export const setList = (objects, marker, isTruncated) => ({
   isTruncated
 })
 
+export const resetList = () => ({
+  type: RESET_LIST
+})
+
 export const appendList = (objects, marker, isTruncated) => ({
   type: APPEND_LIST,
   objects,
@@ -54,47 +56,54 @@ export const appendList = (objects, marker, isTruncated) => ({
 
 export const fetchObjects = append => {
   return function(dispatch, getState) {
-    const {
-      buckets: { currentBucket },
-      objects: { currentPrefix, marker }
-    } = getState()
+    const {buckets: {currentBucket}, objects: {currentPrefix, marker}} = getState()
     if (currentBucket) {
       return web
-      .ListObjects({
-        bucketName: currentBucket,
-        prefix: currentPrefix,
-        marker: append ? marker : ""
-      })
-      .then(res => {
-        let objects = []
-        if (res.objects) {
-          objects = res.objects.map(object => {
-            return {
-              ...object,
-              name: object.name.replace(currentPrefix, "")
-            }
-          })
-        }
-        if (append) {
-          dispatch(appendList(objects, res.nextmarker, res.istruncated))
-        } else {
-          dispatch(setList(objects, res.nextmarker, res.istruncated))
-          dispatch(setSortBy(""))
-          dispatch(setSortOrder(false))
-        }
-        dispatch(setPrefixWritable(res.writable))
-      })
-      .catch(err => {
-        dispatch(alertActions.set({ type: "danger", message: err.message }))
-        history.push("/login")
-      })
-    } 
+        .ListObjects({
+          bucketName: currentBucket,
+          prefix: currentPrefix,
+          marker: append ? marker : ""
+        })
+        .then(res => {
+          let objects = []
+          if (res.objects) {
+            objects = res.objects.map(object => {
+              return {
+                ...object,
+                name: object.name.replace(currentPrefix, "")
+              }
+            })
+          }
+          if (append) {
+            dispatch(appendList(objects, res.nextmarker, res.istruncated))
+          } else {
+            dispatch(setList(objects, res.nextmarker, res.istruncated))
+            dispatch(setSortBy(""))
+            dispatch(setSortOrder(false))
+          }
+          dispatch(setPrefixWritable(res.writable))
+        })
+        .catch(err => {
+          if (web.LoggedIn()) {
+            dispatch(
+              alertActions.set({
+                type: "danger",
+                message: err.message,
+                autoClear: true
+              })
+            )
+            dispatch(resetList())
+          } else {
+            history.push("/login")
+          }
+        })
+    }
   }
 }
 
 export const sortObjects = sortBy => {
   return function(dispatch, getState) {
-    const { objects } = getState()
+    const {objects} = getState()
     const sortOrder = objects.sortBy == sortBy ? !objects.sortOrder : true
     dispatch(setSortBy(sortBy))
     dispatch(setSortOrder(sortOrder))
@@ -194,30 +203,39 @@ export const shareObject = (object, days, hours, minutes) => {
     const currentPrefix = getCurrentPrefix(getState())
     const objectName = `${currentPrefix}${object}`
     const expiry = days * 24 * 60 * 60 + hours * 60 * 60 + minutes * 60
-    return web
-      .PresignedGet({
-        host: location.host,
-        bucket: currentBucket,
-        object: objectName,
-        expiry
-      })
-      .then(obj => {
-        dispatch(showShareObject(object, obj.url))
-        dispatch(
-          alertActions.set({
-            type: "success",
-            message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`
-          })
-        )
-      })
-      .catch(err => {
-        dispatch(
-          alertActions.set({
-            type: "danger",
-            message: err.message
-          })
-        )
-      })
+    if (web.LoggedIn()) {
+      return web
+        .PresignedGet({
+          host: location.host,
+          bucket: currentBucket,
+          object: objectName
+        })
+        .then(obj => {
+          dispatch(showShareObject(object, obj.url))
+          dispatch(
+            alertActions.set({
+              type: "success",
+              message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`
+            })
+          )
+        })
+        .catch(err => {
+          dispatch(
+            alertActions.set({
+              type: "danger",
+              message: err.message
+            })
+          )
+        })
+    } else {
+      dispatch(showShareObject(object, `${location.host}` + '/' + `${currentBucket}` + '/' + encodeURI(objectName)))
+      dispatch(
+        alertActions.set({
+          type: "success",
+          message: `Object shared.`
+        })
+      )
+    }
   }
 }
 
@@ -263,7 +281,7 @@ export const downloadObject = object => {
     } else {
       const url = `${
         window.location.origin
-      }${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=''`
+      }${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=`
       window.location = url
     }
   }
@@ -292,7 +310,7 @@ export const downloadCheckedObjects = () => {
       objects: getCheckedList(state)
     }
     if (!web.LoggedIn()) {
-      const requestUrl = location.origin + "/minio/zip?token=''"
+      const requestUrl = location.origin + "/minio/zip?token="
       downloadZip(requestUrl, req, dispatch)
     } else {
       return web
@@ -303,14 +321,13 @@ export const downloadCheckedObjects = () => {
           }${minioBrowserPrefix}/zip?token=${res.token}`
           downloadZip(requestUrl, req, dispatch)
         })
-        .catch(err =>
-          dispatch(
-            alertActions.set({
-              type: "danger",
-              message: err.message
-            })
-          )
+        .catch(err => dispatch(
+          alertActions.set({
+            type: "danger",
+            message: err.message
+          })
         )
+      )
     }
   }
 }
@@ -333,8 +350,7 @@ const downloadZip = (url, req, dispatch) => {
       var separator = req.prefix.length > 1 ? "-" : ""
 
       anchor.href = blobUrl
-      anchor.download =
-        req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
+      anchor.download = req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
 
       anchor.click()
       window.URL.revokeObjectURL(blobUrl)

browser/app/js/objects/reducer.js

@@ -50,6 +50,13 @@ export default (
         marker: action.marker,
         isTruncated: action.isTruncated
       }
+    case actionsObjects.RESET_LIST:
+      return {
+        ...state,
+        list: [],
+        marker: "",
+        isTruncated: false
+      }
     case actionsObjects.APPEND_LIST:
       return {
         ...state,

browser/app/less/inc/list.less

@@ -74,7 +74,7 @@ div.fesl-row {
     border-bottom: 1px solid transparent;
     cursor: default;
     .transition(background-color);
-    .transition-duration(300ms);
+    .transition-duration(500ms);
 
     @media (max-width: (@screen-xs-max - 100px)) {
         padding: 5px 20px;
@@ -87,15 +87,37 @@ div.fesl-row {
     }
 
     &:hover {
-        &:not(.fesl-row-selected) {
-            background: lighten(@text-muted-color, 22%);
+        .fis-icon {
+            &:before {
+                .opacity(0)
+            }
+        }
+
+        .fis-helper {
+            &:before {
+                .opacity(1);
+            }
+        }
+    }
+
+    .fesl-item-name {
+        a {
+            cursor: default;
         }
     }
 
     /*--------------------------
         Icons
     ----------------------------*/
-    &[data-type=folder] { .list-type(#a1d6dd, '\f114'); }
+    &[data-type=folder] {
+        .list-type(#a1d6dd, '\f114');
+
+        .fesl-item-name {
+            a {
+                cursor: pointer;
+            }
+        }
+    }
     &[data-type=pdf] {.list-type(#fa7775, '\f1c1'); }
     &[data-type=zip] { .list-type(#427089, '\f1c6'); }
     &[data-type=audio] { .list-type(#009688, '\f1c7'); }
@@ -129,14 +151,6 @@ div.fesl-row {
 .fesl-row-selected {
     background-color: @list-row-selected-bg;
 
-    &:nth-child(even) {
-        background-color: @list-row-selected-even-bg;
-    }
-
-    &:hover {
-        background: @list-row-selected-hover;
-    }
-
     &, .fesl-item a {
         color: darken(@text-color, 10%);
     }
@@ -161,6 +175,27 @@ div.fesl-row {
         height: 35px;
         z-index: 8;
         opacity: 0;
+        cursor: pointer;
+
+        &:checked {
+            & ~ .fis-icon {
+                background-color: #32393F;
+
+                &:before {
+                    opacity: 0;
+                }
+            }
+
+            & ~ .fis-helper {
+                &:before {
+                    .scale(0);
+                }
+
+                &:after {
+                    .scale(1);
+                }
+            }
+        }
     }
 }
 
@@ -189,6 +224,38 @@ div.fesl-row {
     }
 }
 
+.fis-helper {
+    &:before,
+    &:after {
+        position: absolute;
+        .transition(all);
+        .transition-duration(250ms);
+    }
+
+    &:before {
+        content: '';
+        width: 15px;
+        height: 15px;
+        border: 2px solid @white;
+        z-index: 7;
+        border-radius: 2px;
+        top: 10px;
+        left: 10px;
+        opacity: 0;
+    }
+
+    &:after {
+        font-family: @font-family-icon;
+        content: '\f00c';
+        top: 8px;
+        left: 9px;
+        color: @white;
+        font-size: 14px;
+        .scale(0);
+    }
+}
+
+
 /*--------------------------
     Files and Folders
 ----------------------------*/

browser/app/less/inc/variables.less

@@ -97,7 +97,5 @@
 /*-------------------------
     List
 --------------------------*/
-@list-row-selected-bg:      #fffad6;
-@list-row-selected-even-bg: #faf5d1;
-@list-row-selected-hover:   #f5f0cc;
+@list-row-selected-bg: #fbf2bf;
 @list-row-even-bg:          #fafafa;

browser/build.js

@@ -70,9 +70,9 @@ async.waterfall([
       commitId = stdout.replace('\n', '')
       if (commitId.length !== 40) throw new Error('commitId invalid : ' + commitId)
       assetsFileName = 'ui-assets.go';
-      var cmd = 'go-bindata-assetfs -pkg browser -nocompress=true production/...'
+      var cmd = 'go-bindata-assetfs -o bindata_assetfs.go -pkg browser -nocompress=true production/...'
       if (!isProduction) {
-        cmd = 'go-bindata-assetfs -pkg browser -nocompress=true dev/...'
+        cmd = 'go-bindata-assetfs -o bindata_assetfs.go -pkg browser -nocompress=true dev/...'
       }
       console.log('Running', cmd)
       exec(cmd, cb)

buildscripts/checkdeps.sh

@@ -34,7 +34,7 @@ _init() {
 
 ## FIXME:
 ## In OSX, 'readlink -f' option does not exist, hence
-## we have our own readlink -f behaviour here.
+## we have our own readlink -f behavior here.
 ## Once OSX has the option, below function is good enough.
 ## 
 ## readlink() {
@@ -89,11 +89,11 @@ check_minimum_version() {
 
 assert_is_supported_arch() {
     case "${ARCH}" in
-        x86_64 | amd64 | aarch64 | arm* )
+        x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x )
             return
             ;;
         *)
-            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, arm*]"
+            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x]"
             exit 1
     esac
 }

buildscripts/gen-ldflags.go

@@ -33,6 +33,7 @@ func genLDFlags(version string) string {
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
 	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
+	ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT")
 	return ldflagsStr
 }
 

buildscripts/go-coverage.sh

@@ -4,7 +4,7 @@ set -e
 echo "" > coverage.txt
 
 for d in $(go list ./... | grep -v browser); do
-    go test -coverprofile=profile.out -covermode=atomic "$d"
+    go test -v -coverprofile=profile.out -covermode=atomic "$d"
     if [ -f profile.out ]; then
         cat profile.out >> coverage.txt
         rm profile.out

buildscripts/verify-build.sh

@@ -71,6 +71,8 @@ function start_minio_erasure_sets()
 function start_minio_dist_erasure_sets()
 {
     declare -a minio_pids
+    export MINIO_ACCESS_KEY=$ACCESS_KEY
+    export MINIO_SECRET_KEY=$SECRET_KEY
     "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
     minio_pids[0]=$!
     "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets4" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets5" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets6" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets7" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets8" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets9" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets10" "http://127.0.0.1:9000${WORK_DIR}/dist-disk-sets11" "http://127.0.0.1:9001${WORK_DIR}/dist-disk-sets12" "http://127.0.0.1:9002${WORK_DIR}/dist-disk-sets13" "http://127.0.0.1:9003${WORK_DIR}/dist-disk-sets14" "http://127.0.0.1:9004${WORK_DIR}/dist-disk-sets15" "http://127.0.0.1:9005${WORK_DIR}/dist-disk-sets16" "http://127.0.0.1:9006${WORK_DIR}/dist-disk-sets17" "http://127.0.0.1:9007${WORK_DIR}/dist-disk-sets18" "http://127.0.0.1:9008${WORK_DIR}/dist-disk-sets19" "http://127.0.0.1:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
@@ -99,6 +101,8 @@ function start_minio_dist_erasure_sets()
 function start_minio_dist_erasure()
 {
     declare -a minio_pids
+    export MINIO_ACCESS_KEY=$ACCESS_KEY
+    export MINIO_SECRET_KEY=$SECRET_KEY
     "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
     minio_pids[0]=$!
     "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &

cmd/acl-handlers.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -54,28 +55,30 @@ type accessControlPolicy struct {
 // This operation uses the ACL
 // subresource to return the ACL of a specified bucket.
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketACL")
+	ctx := newContext(r, w, "GetBucketACL")
+
+	defer logger.AuditLog(w, r, "GetBucketACL", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allow getBucketACL if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -89,7 +92,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -101,7 +104,9 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 // This operation uses the ACL
 // subresource to return the ACL of a specified object.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetObjectACL")
+	ctx := newContext(r, w, "GetObjectACL")
+
+	defer logger.AuditLog(w, r, "GetObjectACL", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -109,21 +114,21 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allow getObjectACL if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Before proceeding validate if object exists.
-	_, err := objAPI.GetObjectInfo(ctx, bucket, object)
+	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -137,7 +142,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/admin-handlers_test.go

@@ -26,8 +26,8 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"os"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -38,100 +38,195 @@ import (
 
 var (
 	configJSON = []byte(`{
-	"version": "13",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
+  "version": "32",
+  "credential": {
+    "accessKey": "minio",
+    "secretKey": "minio123"
+  },
+  "region": "us-east-1",
+  "worm": "off",
+  "storageclass": {
+    "standard": "",
+    "rrs": ""
+  },
+  "cache": {
+    "drives": [],
+    "expiry": 90,
+    "maxuse": 80,
+    "exclude": []
+  },
+  "kms": {
+    "vault": {
+      "endpoint": "",
+      "auth": {
+        "type": "",
+        "approle": {
+          "id": "",
+          "secret": ""
+        }
+      },
+      "key-id": {
+        "name": "",
+        "version": 0
+      }
+    }
+  },
+  "notify": {
+    "amqp": {
+      "1": {
+        "enable": false,
+        "url": "",
+        "exchange": "",
+        "routingKey": "",
+        "exchangeType": "",
+        "deliveryMode": 0,
+        "mandatory": false,
+        "immediate": false,
+        "durable": false,
+        "internal": false,
+        "noWait": false,
+        "autoDeleted": false
+      }
+    },
+    "elasticsearch": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "url": "",
+        "index": ""
+      }
+    },
+    "kafka": {
+      "1": {
+        "enable": false,
+        "brokers": null,
+        "topic": "",
+        "tls": {
+          "enable": false,
+          "skipVerify": false,
+          "clientAuth": 0
+        },
+        "sasl": {
+          "enable": false,
+          "username": "",
+          "password": ""
+        }
+      }
+    },
+    "mqtt": {
+      "1": {
+        "enable": false,
+        "broker": "",
+        "topic": "",
+        "qos": 0,
+        "clientId": "",
+        "username": "",
+        "password": "",
+        "reconnectInterval": 0,
+        "keepAliveInterval": 0
+      }
+    },
+    "mysql": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "dsnString": "",
+        "table": "",
+        "host": "",
+        "port": "",
+        "user": "",
+        "password": "",
+        "database": ""
+      }
+    },
+    "nats": {
+      "1": {
+        "enable": false,
+        "address": "",
+        "subject": "",
+        "username": "",
+        "password": "",
+        "token": "",
+        "secure": false,
+        "pingInterval": 0,
+        "streaming": {
+          "enable": false,
+          "clusterID": "",
+          "clientID": "",
+          "async": false,
+          "maxPubAcksInflight": 0
+        }
+      }
 	},
-	"region": "us-west-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "fatal"
-		},
-		"file": {
+    "nsq": {
+      "1": {
+        "enable": false,
+        "nsqdAddress": "",
+        "topic": "",
+        "tls": {
 			"enable": false,
-			"fileName": "",
-			"level": ""
+			"skipVerify": false
 		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	}
-}`)
+      }
+    },
+    "postgresql": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "connectionString": "",
+        "table": "",
+        "host": "",
+        "port": "",
+        "user": "",
+        "password": "",
+        "database": ""
+      }
+    },
+    "redis": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "address": "",
+        "password": "",
+        "key": ""
+      }
+    },
+    "webhook": {
+      "1": {
+        "enable": false,
+        "endpoint": ""
+      }
+    }
+  },
+  "logger": {
+    "console": {
+      "enabled": true
+    },
+    "http": {
+      "1": {
+        "enabled": false,
+        "endpoint": "https://username:password@example.com/api"
+      }
+    }
+  },
+  "compress": {
+    "enabled": false,
+    "extensions":[".txt",".log",".csv",".json"],
+    "mime-types":["text/csv","text/plain","application/json"]
+  },
+  "openid": {
+    "jwks": {
+      "url": ""
+    }
+  },
+  "policy": {
+    "opa": {
+      "url": "",
+      "authToken": ""
+    }
+  }
+}
+`)
 )
 
 // adminXLTestBed - encapsulates subsystems that need to be setup for
@@ -149,17 +244,17 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// reset global variables to start afresh.
 	resetTestGlobals()
 
-	// Initialize minio server config.
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		return nil, err
-	}
 	// Initializing objectLayer for HealFormatHandler.
 	objLayer, xlDirs, xlErr := initTestXLObjLayer()
 	if xlErr != nil {
 		return nil, xlErr
 	}
 
+	// Initialize minio server config.
+	if err := newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
+		return nil, err
+	}
+
 	// Initialize boot time
 	globalBootTime = UTCNow()
 
@@ -176,10 +271,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// Init global heal state
 	initAllHealState(globalIsXL)
 
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
-	if err != nil {
-		return nil, err
-	}
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
@@ -189,17 +281,15 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	registerAdminRouter(adminRouter)
 
 	return &adminXLTestBed{
-		configPath: rootPath,
-		xlDirs:     xlDirs,
-		objLayer:   objLayer,
-		router:     adminRouter,
+		xlDirs:   xlDirs,
+		objLayer: objLayer,
+		router:   adminRouter,
 	}, nil
 }
 
 // TearDown - method that resets the test bed for subsequent unit
 // tests to start afresh.
 func (atb *adminXLTestBed) TearDown() {
-	os.RemoveAll(atb.configPath)
 	removeRoots(atb.xlDirs)
 	resetTestGlobals()
 }
@@ -219,8 +309,8 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 		for i := 0; i < 10; i++ {
 			objectName := fmt.Sprintf("%s-%d", objName, i)
 			_, err = atb.objLayer.PutObject(context.Background(), bucketName, objectName,
-				mustGetHashReader(t, bytes.NewReader([]byte("hello")),
-					int64(len("hello")), "", ""), nil)
+				mustGetPutObjReader(t, bytes.NewReader([]byte("hello")),
+					int64(len("hello")), "", ""), nil, ObjectOptions{})
 			if err != nil {
 				t.Fatalf("Failed to create %s - %v", objectName,
 					err)
@@ -232,14 +322,14 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 	{
 		objName := "mpObject"
 		uploadID, err := atb.objLayer.NewMultipartUpload(context.Background(), bucketName,
-			objName, nil)
+			objName, nil, ObjectOptions{})
 		if err != nil {
 			t.Fatalf("mp new error: %v", err)
 		}
 
 		_, err = atb.objLayer.PutObjectPart(context.Background(), bucketName, objName,
-			uploadID, 3, mustGetHashReader(t, bytes.NewReader(
-				[]byte("hello")), int64(len("hello")), "", ""))
+			uploadID, 3, mustGetPutObjReader(t, bytes.NewReader(
+				[]byte("hello")), int64(len("hello")), "", ""), ObjectOptions{})
 		if err != nil {
 			t.Fatalf("mp put error: %v", err)
 		}
@@ -418,6 +508,8 @@ func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*htt
 
 	// Set body
 	req.Body = ioutil.NopCloser(bytes.NewReader(body))
+	req.ContentLength = int64(len(body))
+
 	// Set sha-sum header
 	req.Header.Set("X-Amz-Content-Sha256", getSHA256Hash(body))
 
@@ -444,15 +536,21 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
+	var wg sync.WaitGroup
+
 	// Setting up a go routine to simulate ServerRouter's
 	// handleServiceSignals for stop and restart commands.
 	if cmd == restartCmd {
-		go testServiceSignalReceiver(cmd, t)
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			testServiceSignalReceiver(cmd, t)
+		}()
 	}
 	credentials := globalServerConfig.GetCredential()
 
 	body, err := json.Marshal(madmin.ServiceAction{
-		cmd.toServiceActionValue()})
+		Action: cmd.toServiceActionValue()})
 	if err != nil {
 		t.Fatalf("JSONify error: %v", err)
 	}
@@ -483,6 +581,9 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 		t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
 			http.StatusOK, rec.Code, string(resp))
 	}
+
+	// Wait until testServiceSignalReceiver() called in a goroutine quits.
+	wg.Wait()
 }
 
 // Test for service status management REST API.
@@ -538,8 +639,14 @@ func TestServiceSetCreds(t *testing.T) {
 		if err != nil {
 			t.Fatalf("JSONify err: %v", err)
 		}
+
+		ebody, err := madmin.EncryptData(credentials.SecretKey, body)
+		if err != nil {
+			t.Fatal(err)
+		}
+
 		// Construct setCreds request
-		req, err := getServiceCmdRequest(setCreds, credentials, body)
+		req, err := getServiceCmdRequest(setCreds, credentials, ebody)
 		if err != nil {
 			t.Fatalf("Failed to build service status request %v", err)
 		}
@@ -570,193 +677,6 @@ func TestServiceSetCreds(t *testing.T) {
 	}
 }
 
-// mkLockQueryVal - helper function to build lock query param.
-func mkLockQueryVal(bucket, prefix, durationStr string) url.Values {
-	qVal := url.Values{}
-	qVal.Set(string(mgmtBucket), bucket)
-	qVal.Set(string(mgmtPrefix), prefix)
-	qVal.Set(string(mgmtLockOlderThan), durationStr)
-	return qVal
-}
-
-// Test for locks list management REST API.
-func TestListLocksHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// Initialize admin peers to make admin RPC calls.
-	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
-
-	testCases := []struct {
-		bucket         string
-		prefix         string
-		duration       string
-		expectedStatus int
-	}{
-		// Test 1 - valid testcase
-		{
-			bucket:         "mybucket",
-			prefix:         "myobject",
-			duration:       "1s",
-			expectedStatus: http.StatusOK,
-		},
-		// Test 2 - invalid duration
-		{
-			bucket:         "mybucket",
-			prefix:         "myprefix",
-			duration:       "invalidDuration",
-			expectedStatus: http.StatusBadRequest,
-		},
-		// Test 3 - invalid bucket name
-		{
-			bucket:         `invalid\\Bucket`,
-			prefix:         "myprefix",
-			duration:       "1h",
-			expectedStatus: http.StatusBadRequest,
-		},
-		// Test 4 - invalid prefix
-		{
-			bucket:         "mybucket",
-			prefix:         `invalid\\Prefix`,
-			duration:       "1h",
-			expectedStatus: http.StatusBadRequest,
-		},
-	}
-
-	for i, test := range testCases {
-		queryVal := mkLockQueryVal(test.bucket, test.prefix, test.duration)
-		req, err := newTestRequest("GET", "/minio/admin/v1/locks?"+queryVal.Encode(), 0, nil)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to construct list locks request - %v", i+1, err)
-		}
-
-		cred := globalServerConfig.GetCredential()
-		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
-		}
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if test.expectedStatus != rec.Code {
-			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
-		}
-	}
-}
-
-// Test for locks clear management REST API.
-func TestClearLocksHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// Initialize admin peers to make admin RPC calls.
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
-
-	testCases := []struct {
-		bucket         string
-		prefix         string
-		duration       string
-		expectedStatus int
-	}{
-		// Test 1 - valid testcase
-		{
-			bucket:         "mybucket",
-			prefix:         "myobject",
-			duration:       "1s",
-			expectedStatus: http.StatusOK,
-		},
-		// Test 2 - invalid duration
-		{
-			bucket:         "mybucket",
-			prefix:         "myprefix",
-			duration:       "invalidDuration",
-			expectedStatus: http.StatusBadRequest,
-		},
-		// Test 3 - invalid bucket name
-		{
-			bucket:         `invalid\\Bucket`,
-			prefix:         "myprefix",
-			duration:       "1h",
-			expectedStatus: http.StatusBadRequest,
-		},
-		// Test 4 - invalid prefix
-		{
-			bucket:         "mybucket",
-			prefix:         `invalid\\Prefix`,
-			duration:       "1h",
-			expectedStatus: http.StatusBadRequest,
-		},
-	}
-
-	for i, test := range testCases {
-		queryVal := mkLockQueryVal(test.bucket, test.prefix, test.duration)
-		req, err := newTestRequest("DELETE", "/minio/admin/v1/locks?"+queryVal.Encode(), 0, nil)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to construct clear locks request - %v", i+1, err)
-		}
-
-		cred := globalServerConfig.GetCredential()
-		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
-		}
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if test.expectedStatus != rec.Code {
-			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
-		}
-	}
-}
-
-// Test for lock query param validation helper function.
-func TestValidateLockQueryParams(t *testing.T) {
-	// reset globals.
-	// this is to make sure that the tests are not affected by modified globals.
-	resetTestGlobals()
-	// initialize NSLock.
-	initNSLock(false)
-	// Sample query values for test cases.
-	allValidVal := mkLockQueryVal("bucket", "prefix", "1s")
-	invalidBucketVal := mkLockQueryVal(`invalid\\Bucket`, "prefix", "1s")
-	invalidPrefixVal := mkLockQueryVal("bucket", `invalid\\Prefix`, "1s")
-	invalidOlderThanVal := mkLockQueryVal("bucket", "prefix", "invalidDuration")
-
-	testCases := []struct {
-		qVals  url.Values
-		apiErr APIErrorCode
-	}{
-		{
-			qVals:  invalidBucketVal,
-			apiErr: ErrInvalidBucketName,
-		},
-		{
-			qVals:  invalidPrefixVal,
-			apiErr: ErrInvalidObjectName,
-		},
-		{
-			qVals:  invalidOlderThanVal,
-			apiErr: ErrInvalidDuration,
-		},
-		{
-			qVals:  allValidVal,
-			apiErr: ErrNone,
-		},
-	}
-
-	for i, test := range testCases {
-		_, _, _, apiErr := validateLockQueryParams(test.qVals)
-		if apiErr != test.apiErr {
-			t.Errorf("Test %d - Expected error %v but received %v", i+1, test.apiErr, apiErr)
-		}
-	}
-}
-
 // buildAdminRequest - helper function to build an admin API request.
 func buildAdminRequest(queryVal url.Values, method, path string,
 	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {
@@ -818,16 +738,18 @@ func TestSetConfigHandler(t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
-	// SetConfigHandler restarts minio setup - need to start a
-	// signal receiver to receive on globalServiceSignalCh.
-	go testServiceSignalReceiver(restartCmd, t)
-
 	// Prepare query params for set-config mgmt REST API.
 	queryVal := url.Values{}
 	queryVal.Set("config", "")
 
+	password := globalServerConfig.GetCredential().SecretKey
+	econfigJSON, err := madmin.EncryptData(password, configJSON)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
-		int64(len(configJSON)), bytes.NewReader(configJSON))
+		int64(len(econfigJSON)), bytes.NewReader(econfigJSON))
 	if err != nil {
 		t.Fatalf("Failed to construct set-config object request - %v", err)
 	}
@@ -835,23 +757,13 @@ func TestSetConfigHandler(t *testing.T) {
 	rec := httptest.NewRecorder()
 	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
-		t.Errorf("Expected to succeed but failed with %d", rec.Code)
-	}
-
-	result := setConfigResult{}
-	err = json.NewDecoder(rec.Body).Decode(&result)
-	if err != nil {
-		t.Fatalf("Failed to decode set config result json %v", err)
-	}
-
-	if !result.Status {
-		t.Error("Expected set-config to succeed, but failed")
+		t.Errorf("Expected to succeed but failed with %d, body: %s", rec.Code, rec.Body)
 	}
 
 	// Check that a very large config file returns an error.
 	{
 		// Make a large enough config string
-		invalidCfg := []byte(strings.Repeat("A", maxConfigJSONSize+1))
+		invalidCfg := []byte(strings.Repeat("A", maxEConfigJSONSize+1))
 		req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
 			int64(len(invalidCfg)), bytes.NewReader(invalidCfg))
 		if err != nil {
@@ -870,7 +782,7 @@ func TestSetConfigHandler(t *testing.T) {
 	// Check that a config with duplicate keys in an object return
 	// error.
 	{
-		invalidCfg := append(configJSON[:len(configJSON)-1], []byte(`, "version": "15"}`)...)
+		invalidCfg := append(econfigJSON[:len(econfigJSON)-1], []byte(`, "version": "15"}`)...)
 		req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
 			int64(len(invalidCfg)), bytes.NewReader(invalidCfg))
 		if err != nil {
@@ -881,7 +793,7 @@ func TestSetConfigHandler(t *testing.T) {
 		adminTestBed.router.ServeHTTP(rec, req)
 		respBody := string(rec.Body.Bytes())
 		if rec.Code != http.StatusBadRequest ||
-			!strings.Contains(respBody, "JSON configuration provided has objects with duplicate keys") {
+			!strings.Contains(respBody, "JSON configuration provided is of incorrect format") {
 			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
 		}
 	}
@@ -955,12 +867,12 @@ func TestToAdminAPIErr(t *testing.T) {
 		// 3. Non-admin API specific error.
 		{
 			err:            errDiskNotFound,
-			expectedAPIErr: toAPIErrorCode(errDiskNotFound),
+			expectedAPIErr: toAPIErrorCode(context.Background(), errDiskNotFound),
 		},
 	}
 
 	for i, test := range testCases {
-		actualErr := toAdminAPIErrCode(test.err)
+		actualErr := toAdminAPIErrCode(context.Background(), test.err)
 		if actualErr != test.expectedAPIErr {
 			t.Errorf("Test %d: Expected %v but received %v",
 				i+1, test.expectedAPIErr, actualErr)
@@ -968,81 +880,6 @@ func TestToAdminAPIErr(t *testing.T) {
 	}
 }
 
-func TestWriteSetConfigResponse(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-	testCases := []struct {
-		status bool
-		errs   []error
-	}{
-		// 1. all nodes returned success.
-		{
-			status: true,
-			errs:   []error{nil, nil, nil, nil},
-		},
-		// 2. some nodes returned errors.
-		{
-			status: false,
-			errs:   []error{errDiskNotFound, nil, errDiskAccessDenied, errFaultyDisk},
-		},
-	}
-
-	testPeers := []adminPeer{
-		{
-			addr: "localhost:9001",
-		},
-		{
-			addr: "localhost:9002",
-		},
-		{
-			addr: "localhost:9003",
-		},
-		{
-			addr: "localhost:9004",
-		},
-	}
-
-	testURL, err := url.Parse("http://dummy.com")
-	if err != nil {
-		t.Fatalf("Failed to parse a place-holder url")
-	}
-
-	var actualResult setConfigResult
-	for i, test := range testCases {
-		rec := httptest.NewRecorder()
-		writeSetConfigResponse(rec, testPeers, test.errs, test.status, testURL)
-		resp := rec.Result()
-		jsonBytes, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			t.Fatalf("Test %d: Failed to read response %v", i+1, err)
-		}
-
-		err = json.Unmarshal(jsonBytes, &actualResult)
-		if err != nil {
-			t.Fatalf("Test %d: Failed to unmarshal json %v", i+1, err)
-		}
-		if actualResult.Status != test.status {
-			t.Errorf("Test %d: Expected status %v but received %v", i+1, test.status, actualResult.Status)
-		}
-		for p, res := range actualResult.NodeResults {
-			if res.Name != testPeers[p].addr {
-				t.Errorf("Test %d: Expected node name %s but received %s", i+1, testPeers[p].addr, res.Name)
-			}
-			expectedErrMsg := fmt.Sprintf("%v", test.errs[p])
-			if res.ErrMsg != expectedErrMsg {
-				t.Errorf("Test %d: Expected error %s but received %s", i+1, expectedErrMsg, res.ErrMsg)
-			}
-			expectedErrSet := test.errs[p] != nil
-			if res.ErrSet != expectedErrSet {
-				t.Errorf("Test %d: Expected ErrSet %v but received %v", i+1, expectedErrSet, res.ErrSet)
-			}
-		}
-	}
-}
-
 func mkHealStartReq(t *testing.T, bucket, prefix string,
 	opts madmin.HealOpts) *http.Request {
 

cmd/admin-heal-ops.go

@@ -20,12 +20,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 // healStatusSummary - overall short summary of a healing sequence
@@ -57,10 +60,10 @@ const (
 var (
 	errHealIdleTimeout      = fmt.Errorf("healing results were not consumed for too long")
 	errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
-	errHealStopSignalled    = fmt.Errorf("heal stop signalled")
+	errHealStopSignalled    = fmt.Errorf("heal stop signaled")
 
-	errFnHealFromAPIErr = func(err error) error {
-		errCode := toAPIErrorCode(err)
+	errFnHealFromAPIErr = func(ctx context.Context, err error) error {
+		errCode := toAPIErrorCode(ctx, err)
 		apiErr := getAPIError(errCode)
 		return fmt.Errorf("Heal internal error: %s: %s",
 			apiErr.Code, apiErr.Description)
@@ -110,6 +113,32 @@ func initAllHealState(isErasureMode bool) {
 	globalAllHealState = allHealState{
 		healSeqMap: make(map[string]*healSequence),
 	}
+
+	go globalAllHealState.periodicHealSeqsClean()
+}
+
+func (ahs *allHealState) periodicHealSeqsClean() {
+	// Launch clean-up routine to remove this heal sequence (after
+	// it ends) from the global state after timeout has elapsed.
+	ticker := time.NewTicker(time.Minute * 5)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			now := UTCNow()
+			ahs.Lock()
+			for path, h := range ahs.healSeqMap {
+				if h.hasEnded() && h.endTime.Add(keepHealSeqStateDuration).Before(now) {
+					delete(ahs.healSeqMap, path)
+				}
+			}
+			ahs.Unlock()
+		case <-globalServiceDoneCh:
+			// server could be restarting - need
+			// to exit immediately
+			return
+		}
+	}
 }
 
 // getHealSequence - Retrieve a heal sequence by path. The second
@@ -121,6 +150,35 @@ func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists b
 	return h, exists
 }
 
+func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
+	var hsp madmin.HealStopSuccess
+	he, exists := ahs.getHealSequence(path)
+	if !exists {
+		hsp = madmin.HealStopSuccess{
+			ClientToken: "invalid",
+			StartTime:   UTCNow(),
+		}
+	} else {
+		hsp = madmin.HealStopSuccess{
+			ClientToken:   he.clientToken,
+			ClientAddress: he.clientAddress,
+			StartTime:     he.startTime,
+		}
+
+		he.stop()
+		for !he.hasEnded() {
+			time.Sleep(1 * time.Second)
+		}
+		ahs.Lock()
+		defer ahs.Unlock()
+		// Heal sequence explicitly stopped, remove it.
+		delete(ahs.healSeqMap, path)
+	}
+
+	b, err := json.Marshal(&hsp)
+	return b, toAdminAPIErrCode(context.Background(), err)
+}
+
 // LaunchNewHealSequence - launches a background routine that performs
 // healing according to the healSequence argument. For each heal
 // sequence, state is stored in the `globalAllHealState`, which is a
@@ -141,20 +199,20 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 			existsAndLive = true
 		}
 	}
+
 	if existsAndLive {
 		// A heal sequence exists on the given path.
 		if h.forceStarted {
-			// stop the running heal sequence - wait for
-			// it to finish.
+			// stop the running heal sequence - wait for it to finish.
 			he.stop()
 			for !he.hasEnded() {
-				time.Sleep(10 * time.Second)
+				time.Sleep(1 * time.Second)
 			}
 		} else {
 			errMsg = "Heal is already running on the given path " +
 				"(use force-start option to stop and start afresh). " +
-				fmt.Sprintf("The heal was started by IP %s at %s",
-					h.clientAddress, h.startTime)
+				fmt.Sprintf("The heal was started by IP %s at %s, token is %s",
+					h.clientAddress, h.startTime.Format(http.TimeFormat), h.clientToken)
 
 			return nil, ErrHealAlreadyRunning, errMsg
 		}
@@ -181,48 +239,13 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	// Launch top-level background heal go-routine
 	go h.healSequenceStart()
 
-	// Launch clean-up routine to remove this heal sequence (after
-	// it ends) from the global state after timeout has elapsed.
-	go func() {
-		var keepStateTimeout <-chan time.Time
-		ticker := time.NewTicker(time.Minute)
-		defer ticker.Stop()
-		everyMinute := ticker.C
-		for {
-			select {
-			// Check every minute if heal sequence has ended.
-			case <-everyMinute:
-				if h.hasEnded() {
-					keepStateTimeout = time.After(keepHealSeqStateDuration)
-					everyMinute = nil
-				}
-
-			// This case does not fire until the heal
-			// sequence completes.
-			case <-keepStateTimeout:
-				// Heal sequence has ended, keep
-				// results state duration has elapsed,
-				// so purge state.
-				ahs.Lock()
-				defer ahs.Unlock()
-				delete(ahs.healSeqMap, h.path)
-				return
-
-			case <-globalServiceDoneCh:
-				// server could be restarting - need
-				// to exit immediately
-				return
-			}
-		}
-	}()
-
 	b, err := json.Marshal(madmin.HealStartSuccess{
 		ClientToken:   h.clientToken,
 		ClientAddress: h.clientAddress,
 		StartTime:     h.startTime,
 	})
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(h.ctx, err)
 		return nil, ErrInternalError, ""
 	}
 	return b, ErrNone, ""
@@ -270,7 +293,7 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 
 	jbytes, err := json.Marshal(h.currentStatus)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(h.ctx, err)
 		return nil, ErrInternalError
 	}
 
@@ -283,12 +306,15 @@ type healSequence struct {
 	// bucket, and prefix on which heal seq. was initiated
 	bucket, objPrefix string
 
-	// path is just bucket + "/" + objPrefix
+	// path is just pathJoin(bucket, objPrefix)
 	path string
 
 	// time at which heal sequence was started
 	startTime time.Time
 
+	// time at which heal sequence has ended
+	endTime time.Time
+
 	// Heal client info
 	clientToken, clientAddress string
 
@@ -301,7 +327,7 @@ type healSequence struct {
 	// current accumulated status of the heal sequence
 	currentStatus healSequenceStatus
 
-	// channel signalled by background routine when traversal has
+	// channel signaled by background routine when traversal has
 	// completed
 	traverseAndHealDoneCh chan error
 
@@ -328,7 +354,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 	return &healSequence{
 		bucket:        bucket,
 		objPrefix:     objPrefix,
-		path:          bucket + "/" + objPrefix,
+		path:          pathJoin(bucket, objPrefix),
 		startTime:     UTCNow(),
 		clientToken:   mustGetUUID(),
 		clientAddress: clientAddr,
@@ -383,7 +409,6 @@ func (h *healSequence) stop() {
 // sequence automatically resumes. The return value indicates if the
 // operation succeeded.
 func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
-
 	// start a timer to keep an upper time limit to find an empty
 	// slot to add the given heal result - if no slot is found it
 	// means that the server is holding the maximum amount of
@@ -441,7 +466,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 	h.currentStatus.updateLock.Unlock()
 
 	// This is a "safe" point for the heal sequence to quit if
-	// signalled externally.
+	// signaled externally.
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
@@ -467,6 +492,7 @@ func (h *healSequence) healSequenceStart() {
 
 	select {
 	case err, ok := <-h.traverseAndHealDoneCh:
+		h.endTime = UTCNow()
 		h.currentStatus.updateLock.Lock()
 		defer h.currentStatus.updateLock.Unlock()
 		// Heal traversal is complete.
@@ -480,6 +506,7 @@ func (h *healSequence) healSequenceStart() {
 		}
 
 	case <-h.stopSignalCh:
+		h.endTime = UTCNow()
 		h.currentStatus.updateLock.Lock()
 		h.currentStatus.Summary = healStoppedStatus
 		h.currentStatus.FailureDetail = errHealStopSignalled.Error()
@@ -519,6 +546,9 @@ func (h *healSequence) traverseAndHeal() {
 	// Start with format healing
 	checkErr(h.healDiskFormat)
 
+	// Start healing the config.
+	checkErr(h.healConfig)
+
 	// Heal buckets and objects
 	checkErr(h.healBuckets)
 
@@ -529,9 +559,72 @@ func (h *healSequence) traverseAndHeal() {
 	close(h.traverseAndHealDoneCh)
 }
 
+// healConfig - heals config.json, retrun value indicates if a failure occurred.
+func (h *healSequence) healConfig() error {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// NOTE: Healing on configs is run regardless
+	// of any bucket being selected, this is to ensure that
+	// configs are always uptodate and correct.
+	marker := ""
+	isTruncated := true
+	for isTruncated {
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to heal
+			waitCount := 60
+			// Any requests in progress, delay the heal.
+			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
+		}
+
+		// Lists all objects under `config` prefix.
+		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, minioMetaBucket, minioConfigPrefix,
+			marker, "", 1000)
+		if err != nil {
+			return errFnHealFromAPIErr(h.ctx, err)
+		}
+
+		for index := range objectInfos.Objects {
+			if h.isQuitting() {
+				return errHealStopSignalled
+			}
+			o := objectInfos.Objects[index]
+			res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun)
+			// Object might have been deleted, by the time heal
+			// was attempted we ignore this file an move on.
+			if isErrObjectNotFound(herr) {
+				continue
+			}
+			if herr != nil {
+				return herr
+			}
+			res.Type = madmin.HealItemBucketMetadata
+			if err = h.pushHealResultItem(res); err != nil {
+				return err
+			}
+		}
+
+		isTruncated = objectInfos.IsTruncated
+		marker = objectInfos.NextMarker
+	}
+
+	return nil
+}
+
 // healDiskFormat - heals format.json, return value indicates if a
 // failure error occurred.
 func (h *healSequence) healDiskFormat() error {
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -542,7 +635,7 @@ func (h *healSequence) healDiskFormat() error {
 	// return any error, ignore error returned when disks have
 	// already healed.
 	if err != nil && err != errNoHealRequired {
-		return errFnHealFromAPIErr(err)
+		return errFnHealFromAPIErr(h.ctx, err)
 	}
 
 	// Healing succeeded notify the peers to reload format and re-initialize disks.
@@ -557,6 +650,10 @@ func (h *healSequence) healDiskFormat() error {
 
 // healBuckets - check for all buckets heal or just particular bucket.
 func (h *healSequence) healBuckets() error {
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
 		return h.healBucket(h.bucket)
@@ -570,7 +667,7 @@ func (h *healSequence) healBuckets() error {
 
 	buckets, err := objectAPI.ListBucketsHeal(h.ctx)
 	if err != nil {
-		return errFnHealFromAPIErr(err)
+		return errFnHealFromAPIErr(h.ctx, err)
 	}
 
 	for _, bucket := range buckets {
@@ -584,10 +681,6 @@ func (h *healSequence) healBuckets() error {
 
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(bucket string) error {
-	if h.isQuitting() {
-		return errHealStopSignalled
-	}
-
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -610,10 +703,9 @@ func (h *healSequence) healBucket(bucket string) error {
 		if h.objPrefix != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			_, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix)
+			_, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{})
 			if err == nil {
-				err = h.healObject(bucket, h.objPrefix)
-				if err != nil {
+				if err = h.healObject(bucket, h.objPrefix); err != nil {
 					return err
 				}
 			}
@@ -622,17 +714,40 @@ func (h *healSequence) healBucket(bucket string) error {
 		return nil
 	}
 
+	entries := runtime.NumCPU() * globalEndpoints.Nodes()
+
 	marker := ""
 	isTruncated := true
 	for isTruncated {
-		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, bucket,
-			h.objPrefix, marker, "", 1000)
-		if err != nil {
-			return errFnHealFromAPIErr(err)
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to heal
+			waitCount := 60
+			// Any requests in progress, delay the heal.
+			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
 		}
 
-		for _, o := range objectInfos.Objects {
-			if err := h.healObject(o.Bucket, o.Name); err != nil {
+		// Heal numCPU * nodes objects at a time.
+		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, bucket,
+			h.objPrefix, marker, "", entries)
+		if err != nil {
+			return errFnHealFromAPIErr(h.ctx, err)
+		}
+
+		g := errgroup.WithNErrs(len(objectInfos.Objects))
+		for index := range objectInfos.Objects {
+			index := index
+			g.Go(func() error {
+				o := objectInfos.Objects[index]
+				return h.healObject(o.Bucket, o.Name)
+			}, index)
+		}
+
+		for _, err := range g.Wait() {
+			if err != nil {
 				return err
 			}
 		}
@@ -656,6 +771,9 @@ func (h *healSequence) healObject(bucket, object string) error {
 	}
 
 	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun)
+	if isErrObjectNotFound(err) {
+		return nil
+	}
 	if err != nil {
 		hri.Detail = err.Error()
 	}

cmd/admin-router.go

@@ -53,26 +53,58 @@ func registerAdminRouter(router *mux.Router) {
 	// Info operations
 	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
 
-	/// Lock operations
+	if globalIsDistXL || globalIsXL {
+		/// Heal operations
 
-	// List Locks
-	adminV1Router.Methods(http.MethodGet).Path("/locks").HandlerFunc(httpTraceAll(adminAPI.ListLocksHandler))
-	// Clear locks
-	adminV1Router.Methods(http.MethodDelete).Path("/locks").HandlerFunc(httpTraceAll(adminAPI.ClearLocksHandler))
+		// Heal processing endpoint.
+		adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+	}
 
-	/// Heal operations
-
-	// Heal processing endpoint.
-	adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+	// Profiling operations
+	adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
+		Queries("profilerType", "{profilerType:.*}")
+	adminV1Router.Methods(http.MethodGet).Path("/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
 
 	/// Config operations
 
 	// Update credentials
-	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceAll(adminAPI.UpdateCredentialsHandler))
+	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
 	// Get config
-	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceAll(adminAPI.GetConfigHandler))
+	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 	// Set config
-	adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceAll(adminAPI.SetConfigHandler))
+	adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
+
+	// Get config keys/values
+	adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
+	// Set config keys/values
+	adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
+
+	// -- IAM APIs --
+
+	// Add policy IAM
+	adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
+
+	// Add user IAM
+	adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
+	adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
+		Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
+	adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
+		Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
+
+	// Remove policy IAM
+	adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+
+	// Remove user IAM
+	adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+
+	// List users
+	adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+
+	// List policies
+	adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+
+	// If none of the routes match.
+	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandler))
 }

cmd/admin-rpc-client.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"context"
 	"crypto/tls"
-	"encoding/json"
 	"fmt"
 	"net"
 	"sort"
@@ -54,19 +53,6 @@ func (rpcClient *AdminRPCClient) ReInitFormat(dryRun bool) error {
 	return rpcClient.Call(adminServiceName+".ReInitFormat", &args, &reply)
 }
 
-// ListLocks - Sends list locks command to remote server via RPC.
-func (rpcClient *AdminRPCClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	args := ListLocksQuery{
-		Bucket:   bucket,
-		Prefix:   prefix,
-		Duration: duration,
-	}
-	var reply []VolumeLockInfo
-
-	err := rpcClient.Call(adminServiceName+".ListLocks", &args, &reply)
-	return reply, err
-}
-
 // ServerInfo - returns the server info of the server to which the RPC call is made.
 func (rpcClient *AdminRPCClient) ServerInfo() (sid ServerInfoData, err error) {
 	err = rpcClient.Call(adminServiceName+".ServerInfo", &AuthArgs{}, &sid)
@@ -82,27 +68,20 @@ func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
 	return reply, err
 }
 
-// WriteTmpConfig - writes config file content to a temporary file on a remote node.
-func (rpcClient *AdminRPCClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	args := WriteConfigArgs{
-		TmpFileName: tmpFileName,
-		Buf:         configBytes,
-	}
+// StartProfiling - starts profiling in the remote server.
+func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
+	args := StartProfilingArgs{Profiler: profiler}
 	reply := VoidReply{}
-
-	err := rpcClient.Call(adminServiceName+".WriteTmpConfig", &args, &reply)
-	logger.LogIf(context.Background(), err)
-	return err
+	return rpcClient.Call(adminServiceName+".StartProfiling", &args, &reply)
 }
 
-// CommitConfig - Move the new config in tmpFileName onto config.json on a remote node.
-func (rpcClient *AdminRPCClient) CommitConfig(tmpFileName string) error {
-	args := CommitConfigArgs{FileName: tmpFileName}
-	reply := VoidReply{}
+// DownloadProfilingData - returns profiling data of the remote server.
+func (rpcClient *AdminRPCClient) DownloadProfilingData() ([]byte, error) {
+	args := AuthArgs{}
+	var reply []byte
 
-	err := rpcClient.Call(adminServiceName+".CommitConfig", &args, &reply)
-	logger.LogIf(context.Background(), err)
-	return err
+	err := rpcClient.Call(adminServiceName+".DownloadProfilingData", &args, &reply)
+	return reply, err
 }
 
 // NewAdminRPCClient - returns new admin RPC client.
@@ -147,11 +126,10 @@ func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
 type adminCmdRunner interface {
 	SignalService(s serviceSignal) error
 	ReInitFormat(dryRun bool) error
-	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
 	ServerInfo() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
-	WriteTmpConfig(tmpFileName string, configBytes []byte) error
-	CommitConfig(tmpFileName string) error
+	StartProfiling(string) error
+	DownloadProfilingData() ([]byte, error)
 }
 
 // adminPeer - represents an entity that implements admin API RPCs.
@@ -179,9 +157,9 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
 
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to parse Admin RPC Host")
 		rpcClient, err := NewAdminRPCClient(host)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to initialize Admin RPC Client")
 		adminPeerList = append(adminPeerList, adminPeer{
 			addr:      hostStr,
 			cmdRunner: rpcClient,
@@ -244,56 +222,6 @@ func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
 	errs[0] = invokeServiceCmd(cps[0], cmd)
 }
 
-// listPeerLocksInfo - fetch list of locks held on the given bucket,
-// matching prefix held longer than duration from all peer servers.
-func listPeerLocksInfo(peers adminPeers, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	// Used to aggregate volume lock information from all nodes.
-	allLocks := make([][]VolumeLockInfo, len(peers))
-	errs := make([]error, len(peers))
-	var wg sync.WaitGroup
-	localPeer := peers[0]
-	remotePeers := peers[1:]
-	for i, remotePeer := range remotePeers {
-		wg.Add(1)
-		go func(idx int, remotePeer adminPeer) {
-			defer wg.Done()
-			// `remotePeers` is right-shifted by one position relative to `peers`
-			allLocks[idx], errs[idx] = remotePeer.cmdRunner.ListLocks(bucket, prefix, duration)
-		}(i+1, remotePeer)
-	}
-	wg.Wait()
-	allLocks[0], errs[0] = localPeer.cmdRunner.ListLocks(bucket, prefix, duration)
-
-	// Summarizing errors received for ListLocks RPC across all
-	// nodes.  N B the possible unavailability of quorum in errors
-	// applies only to distributed setup.
-	errCount, err := reduceErrs(errs, []error{})
-	if err != nil {
-		if errCount >= (len(peers)/2 + 1) {
-			return nil, err
-		}
-		return nil, InsufficientReadQuorum{}
-	}
-
-	// Group lock information across nodes by (bucket, object)
-	// pair. For readability only.
-	paramLockMap := make(map[nsParam][]VolumeLockInfo)
-	for _, nodeLocks := range allLocks {
-		for _, lockInfo := range nodeLocks {
-			param := nsParam{
-				volume: lockInfo.Bucket,
-				path:   lockInfo.Object,
-			}
-			paramLockMap[param] = append(paramLockMap[param], lockInfo)
-		}
-	}
-	groupedLockInfos := []VolumeLockInfo{}
-	for _, volLocks := range paramLockMap {
-		groupedLockInfos = append(groupedLockInfos, volLocks...)
-	}
-	return groupedLockInfos, nil
-}
-
 // uptimeSlice - used to sort uptimes in chronological order.
 type uptimeSlice []struct {
 	err    error
@@ -365,187 +293,3 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
 
 	return latestUptime, nil
 }
-
-// getPeerConfig - Fetches config.json from all nodes in the setup and
-// returns the one that occurs in a majority of them.
-func getPeerConfig(peers adminPeers) ([]byte, error) {
-	if !globalIsDistXL {
-		return peers[0].cmdRunner.GetConfig()
-	}
-
-	errs := make([]error, len(peers))
-	configs := make([][]byte, len(peers))
-
-	// Get config from all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			configs[idx], errs[idx] = peer.cmdRunner.GetConfig()
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Find the maximally occurring config among peers in a
-	// distributed setup.
-
-	serverConfigs := make([]serverConfig, len(peers))
-	for i, configBytes := range configs {
-		if errs[i] != nil {
-			continue
-		}
-
-		// Unmarshal the received config files.
-		err := json.Unmarshal(configBytes, &serverConfigs[i])
-		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peers[i].addr)
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-	}
-
-	configJSON, err := getValidServerConfig(serverConfigs, errs)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return nil, err
-	}
-
-	// Return the config.json that was present quorum or more
-	// number of disks.
-	return json.Marshal(configJSON)
-}
-
-// getValidServerConfig - finds the server config that is present in
-// quorum or more number of servers.
-func getValidServerConfig(serverConfigs []serverConfig, errs []error) (scv serverConfig, e error) {
-	// majority-based quorum
-	quorum := len(serverConfigs)/2 + 1
-
-	// Count the number of disks a config.json was found in.
-	configCounter := make([]int, len(serverConfigs))
-
-	// We group equal serverConfigs by the lowest index of the
-	// same value;  e.g, let us take the following serverConfigs
-	// in a 4-node setup,
-	// serverConfigs == [c1, c2, c1, c1]
-	// configCounter == [3, 1, 0, 0]
-	// c1, c2 are the only distinct values that appear.  c1 is
-	// identified by 0, the lowest index it appears in and c2 is
-	// identified by 1. So, we need to find the number of times
-	// each of these distinct values occur.
-
-	// Invariants:
-
-	// 1. At the beginning of the i-th iteration, the number of
-	// unique configurations seen so far is equal to the number of
-	// non-zero counter values in config[:i].
-
-	// 2. At the beginning of the i-th iteration, the sum of
-	// elements of configCounter[:i] is equal to the number of
-	// non-error configurations seen so far.
-
-	// For each of the serverConfig ...
-	for i := range serverConfigs {
-		// Skip nodes where getConfig failed.
-		if errs[i] != nil {
-			continue
-		}
-		// Check if it is equal to any of the configurations
-		// seen so far. If j == i is reached then we have an
-		// unseen configuration.
-		for j := 0; j <= i; j++ {
-			if j < i && configCounter[j] == 0 {
-				// serverConfigs[j] is known to be
-				// equal to a value that was already
-				// seen. See example above for
-				// clarity.
-				continue
-			} else if j < i && serverConfigs[i].ConfigDiff(&serverConfigs[j]) == "" {
-				// serverConfigs[i] is equal to
-				// serverConfigs[j], update
-				// serverConfigs[j]'s counter since it
-				// is the lower index.
-				configCounter[j]++
-				break
-			} else if j == i {
-				// serverConfigs[i] is equal to no
-				// other value seen before. It is
-				// unique so far.
-				configCounter[i] = 1
-				break
-			} // else invariants specified above are violated.
-		}
-	}
-
-	// We find the maximally occurring server config and check if
-	// there is quorum.
-	var configJSON serverConfig
-	maxOccurrence := 0
-	for i, count := range configCounter {
-		if maxOccurrence < count {
-			maxOccurrence = count
-			configJSON = serverConfigs[i]
-		}
-	}
-
-	// If quorum nodes don't agree.
-	if maxOccurrence < quorum {
-		return scv, errXLWriteQuorum
-	}
-
-	return configJSON, nil
-}
-
-// Write config contents into a temporary file on all nodes.
-func writeTmpConfigPeers(peers adminPeers, tmpFileName string, configBytes []byte) []error {
-	// For a single-node minio server setup.
-	if !globalIsDistXL {
-		err := peers[0].cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
-		return []error{err}
-	}
-
-	errs := make([]error, len(peers))
-
-	// Write config into temporary file on all nodes.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			errs[idx] = peer.cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Return bytes written and errors (if any) during writing
-	// temporary config file.
-	return errs
-}
-
-// Move config contents from the given temporary file onto config.json
-// on all nodes.
-func commitConfigPeers(peers adminPeers, tmpFileName string) []error {
-	// For a single-node minio server setup.
-	if !globalIsDistXL {
-		return []error{peers[0].cmdRunner.CommitConfig(tmpFileName)}
-	}
-
-	errs := make([]error, len(peers))
-
-	// Rename temporary config file into configDir/config.json on
-	// all nodes.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			errs[idx] = peer.cmdRunner.CommitConfig(tmpFileName)
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Return errors (if any) received during rename.
-	return errs
-}

cmd/admin-rpc-client.go.orig

@@ -1,650 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014, 2015, 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net"
-	"os"
-	"path"
-	"path/filepath"
-	"sort"
-	"sync"
-	"time"
-
-	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/cmd/logger"
-)
-
-const (
-	// Admin service names
-	signalServiceRPC  = "Admin.SignalService"
-	reInitFormatRPC   = "Admin.ReInitFormat"
-	listLocksRPC      = "Admin.ListLocks"
-	serverInfoDataRPC = "Admin.ServerInfoData"
-	getConfigRPC      = "Admin.GetConfig"
-	writeTmpConfigRPC = "Admin.WriteTmpConfig"
-	commitConfigRPC   = "Admin.CommitConfig"
-)
-
-// localAdminClient - represents admin operation to be executed locally.
-type localAdminClient struct {
-}
-
-// remoteAdminClient - represents admin operation to be executed
-// remotely, via RPC.
-type remoteAdminClient struct {
-	*AuthRPCClient
-}
-
-// adminCmdRunner - abstracts local and remote execution of admin
-// commands like service stop and service restart.
-type adminCmdRunner interface {
-	SignalService(s serviceSignal) error
-	ReInitFormat(dryRun bool) error
-	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
-	ServerInfoData() (ServerInfoData, error)
-	GetConfig() ([]byte, error)
-	WriteTmpConfig(tmpFileName string, configBytes []byte) error
-	CommitConfig(tmpFileName string) error
-}
-
-var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
-
-// SignalService - sends a restart or stop signal to the local server
-func (lc localAdminClient) SignalService(s serviceSignal) error {
-	switch s {
-	case serviceRestart, serviceStop:
-		globalServiceSignalCh <- s
-	default:
-		return errUnsupportedSignal
-	}
-	return nil
-}
-
-// ReInitFormat - re-initialize disk format.
-func (lc localAdminClient) ReInitFormat(dryRun bool) error {
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	return objectAPI.ReloadFormat(context.Background(), dryRun)
-}
-
-// ListLocks - Fetches lock information from local lock instrumentation.
-func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	// check if objectLayer is initialized, if not return.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return nil, errServerNotInitialized
-	}
-	return objectAPI.ListLocks(context.Background(), bucket, prefix, duration)
-}
-
-func (rc remoteAdminClient) SignalService(s serviceSignal) (err error) {
-	switch s {
-	case serviceRestart, serviceStop:
-		reply := AuthRPCReply{}
-		err = rc.Call(signalServiceRPC, &SignalServiceArgs{Sig: s},
-			&reply)
-	default:
-		err = errUnsupportedSignal
-	}
-	return err
-}
-
-// ReInitFormat - re-initialize disk format, remotely.
-func (rc remoteAdminClient) ReInitFormat(dryRun bool) error {
-	reply := AuthRPCReply{}
-	return rc.Call(reInitFormatRPC, &ReInitFormatArgs{
-		DryRun: dryRun,
-	}, &reply)
-}
-
-// ListLocks - Sends list locks command to remote server via RPC.
-func (rc remoteAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	listArgs := ListLocksQuery{
-		Bucket:   bucket,
-		Prefix:   prefix,
-		Duration: duration,
-	}
-	var reply ListLocksReply
-	if err := rc.Call(listLocksRPC, &listArgs, &reply); err != nil {
-		return nil, err
-	}
-	return reply.VolLocks, nil
-}
-
-// ServerInfoData - Returns the server info of this server.
-func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	if globalBootTime.IsZero() {
-		return sid, errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return sid, errServerNotInitialized
-	}
-	storage := objLayer.StorageInfo(context.Background())
-
-	return ServerInfoData{
-		StorageInfo: storage,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			SQSARN:   globalNotificationSys.GetARNList(),
-			Region:   globalServerConfig.GetRegion(),
-		},
-	}, nil
-}
-
-// ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	args := AuthRPCArgs{}
-	reply := ServerInfoDataReply{}
-	err := rc.Call(serverInfoDataRPC, &args, &reply)
-	if err != nil {
-		return sid, err
-	}
-
-	return reply.ServerInfoData, nil
-}
-
-// GetConfig - returns config.json of the local server.
-func (lc localAdminClient) GetConfig() ([]byte, error) {
-	if globalServerConfig == nil {
-		return nil, fmt.Errorf("config not present")
-	}
-
-	return json.Marshal(globalServerConfig)
-}
-
-// GetConfig - returns config.json of the remote server.
-func (rc remoteAdminClient) GetConfig() ([]byte, error) {
-	args := AuthRPCArgs{}
-	reply := ConfigReply{}
-	if err := rc.Call(getConfigRPC, &args, &reply); err != nil {
-		return nil, err
-	}
-	return reply.Config, nil
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// the local server.
-func (lc localAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	return writeTmpConfigCommon(tmpFileName, configBytes)
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// a remote node.
-func (rc remoteAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	wArgs := WriteConfigArgs{
-		TmpFileName: tmpFileName,
-		Buf:         configBytes,
-	}
-
-	err := rc.Call(writeTmpConfigRPC, &wArgs, &WriteConfigReply{})
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-
-	return nil
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a local node.
-func (lc localAdminClient) CommitConfig(tmpFileName string) error {
-	configFile := getConfigFile()
-	tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName)
-
-	err := os.Rename(tmpConfigFile, configFile)
-	reqInfo := (&logger.ReqInfo{}).AppendTags("tmpConfigFile", tmpConfigFile)
-	reqInfo.AppendTags("configFile", configFile)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
-	logger.LogIf(ctx, err)
-	return err
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a remote node.
-func (rc remoteAdminClient) CommitConfig(tmpFileName string) error {
-	cArgs := CommitConfigArgs{
-		FileName: tmpFileName,
-	}
-	cReply := CommitConfigReply{}
-	err := rc.Call(commitConfigRPC, &cArgs, &cReply)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-
-	return nil
-}
-
-// adminPeer - represents an entity that implements admin API RPCs.
-type adminPeer struct {
-	addr      string
-	cmdRunner adminCmdRunner
-	isLocal   bool
-}
-
-// type alias for a collection of adminPeer.
-type adminPeers []adminPeer
-
-// makeAdminPeers - helper function to construct a collection of adminPeer.
-func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
-	thisPeer := globalMinioAddr
-	if globalMinioHost == "" {
-		// When host is not explicitly provided simply
-		// use the first IPv4.
-		thisPeer = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
-	}
-	adminPeerList = append(adminPeerList, adminPeer{
-		thisPeer,
-		localAdminClient{},
-		true,
-	})
-
-	hostSet := set.CreateStringSet(globalMinioAddr)
-	cred := globalServerConfig.GetCredential()
-	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
-	for _, host := range GetRemotePeers(endpoints) {
-		if hostSet.Contains(host) {
-			continue
-		}
-		hostSet.Add(host)
-		adminPeerList = append(adminPeerList, adminPeer{
-			addr: host,
-			cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
-				accessKey:       cred.AccessKey,
-				secretKey:       cred.SecretKey,
-				serverAddr:      host,
-				serviceEndpoint: serviceEndpoint,
-				secureConn:      globalIsSSL,
-				serviceName:     "Admin",
-			})},
-		})
-	}
-
-	return adminPeerList
-}
-
-// peersReInitFormat - reinitialize remote object layers to new format.
-func peersReInitFormat(peers adminPeers, dryRun bool) error {
-	errs := make([]error, len(peers))
-
-	// Send ReInitFormat RPC call to all nodes.
-	// for local adminPeer this is a no-op.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			if !peer.isLocal {
-				errs[idx] = peer.cmdRunner.ReInitFormat(dryRun)
-			}
-		}(i, peer)
-	}
-	wg.Wait()
-	return nil
-}
-
-// Initialize global adminPeer collection.
-func initGlobalAdminPeers(endpoints EndpointList) {
-	globalAdminPeers = makeAdminPeers(endpoints)
-}
-
-// invokeServiceCmd - Invoke Restart/Stop command.
-func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
-	switch cmd {
-	case serviceRestart, serviceStop:
-		err = cp.cmdRunner.SignalService(cmd)
-	}
-	return err
-}
-
-// sendServiceCmd - Invoke Restart command on remote peers
-// adminPeer followed by on the local peer.
-func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
-	// Send service command like stop or restart to all remote nodes and finally run on local node.
-	errs := make([]error, len(cps))
-	var wg sync.WaitGroup
-	remotePeers := cps[1:]
-	for i := range remotePeers {
-		wg.Add(1)
-		go func(idx int) {
-			defer wg.Done()
-			// we use idx+1 because remotePeers slice is 1 position shifted w.r.t cps
-			errs[idx+1] = invokeServiceCmd(remotePeers[idx], cmd)
-		}(i)
-	}
-	wg.Wait()
-	errs[0] = invokeServiceCmd(cps[0], cmd)
-}
-
-// listPeerLocksInfo - fetch list of locks held on the given bucket,
-// matching prefix held longer than duration from all peer servers.
-func listPeerLocksInfo(peers adminPeers, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	// Used to aggregate volume lock information from all nodes.
-	allLocks := make([][]VolumeLockInfo, len(peers))
-	errs := make([]error, len(peers))
-	var wg sync.WaitGroup
-	localPeer := peers[0]
-	remotePeers := peers[1:]
-	for i, remotePeer := range remotePeers {
-		wg.Add(1)
-		go func(idx int, remotePeer adminPeer) {
-			defer wg.Done()
-			// `remotePeers` is right-shifted by one position relative to `peers`
-			allLocks[idx], errs[idx] = remotePeer.cmdRunner.ListLocks(bucket, prefix, duration)
-		}(i+1, remotePeer)
-	}
-	wg.Wait()
-	allLocks[0], errs[0] = localPeer.cmdRunner.ListLocks(bucket, prefix, duration)
-
-	// Summarizing errors received for ListLocks RPC across all
-	// nodes.  N B the possible unavailability of quorum in errors
-	// applies only to distributed setup.
-	errCount, err := reduceErrs(errs, []error{})
-	if err != nil {
-		if errCount >= (len(peers)/2 + 1) {
-			return nil, err
-		}
-		return nil, InsufficientReadQuorum{}
-	}
-
-	// Group lock information across nodes by (bucket, object)
-	// pair. For readability only.
-	paramLockMap := make(map[nsParam][]VolumeLockInfo)
-	for _, nodeLocks := range allLocks {
-		for _, lockInfo := range nodeLocks {
-			param := nsParam{
-				volume: lockInfo.Bucket,
-				path:   lockInfo.Object,
-			}
-			paramLockMap[param] = append(paramLockMap[param], lockInfo)
-		}
-	}
-	groupedLockInfos := []VolumeLockInfo{}
-	for _, volLocks := range paramLockMap {
-		groupedLockInfos = append(groupedLockInfos, volLocks...)
-	}
-	return groupedLockInfos, nil
-}
-
-// uptimeSlice - used to sort uptimes in chronological order.
-type uptimeSlice []struct {
-	err    error
-	uptime time.Duration
-}
-
-func (ts uptimeSlice) Len() int {
-	return len(ts)
-}
-
-func (ts uptimeSlice) Less(i, j int) bool {
-	return ts[i].uptime < ts[j].uptime
-}
-
-func (ts uptimeSlice) Swap(i, j int) {
-	ts[i], ts[j] = ts[j], ts[i]
-}
-
-// getPeerUptimes - returns the uptime since the last time read quorum
-// was established on success. Otherwise returns errXLReadQuorum.
-func getPeerUptimes(peers adminPeers) (time.Duration, error) {
-	// In a single node Erasure or FS backend setup the uptime of
-	// the setup is the uptime of the single minio server
-	// instance.
-	if !globalIsDistXL {
-		return UTCNow().Sub(globalBootTime), nil
-	}
-
-	uptimes := make(uptimeSlice, len(peers))
-
-	// Get up time of all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
-			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Sort uptimes in chronological order.
-	sort.Sort(uptimes)
-
-	// Pick the readQuorum'th uptime in chronological order. i.e,
-	// the time at which read quorum was (re-)established.
-	readQuorum := len(uptimes) / 2
-	validCount := 0
-	latestUptime := time.Duration(0)
-	for _, uptime := range uptimes {
-		if uptime.err != nil {
-			logger.LogIf(context.Background(), uptime.err)
-			continue
-		}
-
-		validCount++
-		if validCount >= readQuorum {
-			latestUptime = uptime.uptime
-			break
-		}
-	}
-
-	// Less than readQuorum "Admin.Uptime" RPC call returned
-	// successfully, so read-quorum unavailable.
-	if validCount < readQuorum {
-		return time.Duration(0), InsufficientReadQuorum{}
-	}
-
-	return latestUptime, nil
-}
-
-// getPeerConfig - Fetches config.json from all nodes in the setup and
-// returns the one that occurs in a majority of them.
-func getPeerConfig(peers adminPeers) ([]byte, error) {
-	if !globalIsDistXL {
-		return peers[0].cmdRunner.GetConfig()
-	}
-
-	errs := make([]error, len(peers))
-	configs := make([][]byte, len(peers))
-
-	// Get config from all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			configs[idx], errs[idx] = peer.cmdRunner.GetConfig()
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Find the maximally occurring config among peers in a
-	// distributed setup.
-
-	serverConfigs := make([]serverConfig, len(peers))
-	for i, configBytes := range configs {
-		if errs[i] != nil {
-			continue
-		}
-
-		// Unmarshal the received config files.
-		err := json.Unmarshal(configBytes, &serverConfigs[i])
-		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peers[i].addr)
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-	}
-
-	configJSON, err := getValidServerConfig(serverConfigs, errs)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return nil, err
-	}
-
-	// Return the config.json that was present quorum or more
-	// number of disks.
-	return json.Marshal(configJSON)
-}
-
-// getValidServerConfig - finds the server config that is present in
-// quorum or more number of servers.
-func getValidServerConfig(serverConfigs []serverConfig, errs []error) (scv serverConfig, e error) {
-	// majority-based quorum
-	quorum := len(serverConfigs)/2 + 1
-
-	// Count the number of disks a config.json was found in.
-	configCounter := make([]int, len(serverConfigs))
-
-	// We group equal serverConfigs by the lowest index of the
-	// same value;  e.g, let us take the following serverConfigs
-	// in a 4-node setup,
-	// serverConfigs == [c1, c2, c1, c1]
-	// configCounter == [3, 1, 0, 0]
-	// c1, c2 are the only distinct values that appear.  c1 is
-	// identified by 0, the lowest index it appears in and c2 is
-	// identified by 1. So, we need to find the number of times
-	// each of these distinct values occur.
-
-	// Invariants:
-
-	// 1. At the beginning of the i-th iteration, the number of
-	// unique configurations seen so far is equal to the number of
-	// non-zero counter values in config[:i].
-
-	// 2. At the beginning of the i-th iteration, the sum of
-	// elements of configCounter[:i] is equal to the number of
-	// non-error configurations seen so far.
-
-	// For each of the serverConfig ...
-	for i := range serverConfigs {
-		// Skip nodes where getConfig failed.
-		if errs[i] != nil {
-			continue
-		}
-		// Check if it is equal to any of the configurations
-		// seen so far. If j == i is reached then we have an
-		// unseen configuration.
-		for j := 0; j <= i; j++ {
-			if j < i && configCounter[j] == 0 {
-				// serverConfigs[j] is known to be
-				// equal to a value that was already
-				// seen. See example above for
-				// clarity.
-				continue
-			} else if j < i && serverConfigs[i].ConfigDiff(&serverConfigs[j]) == "" {
-				// serverConfigs[i] is equal to
-				// serverConfigs[j], update
-				// serverConfigs[j]'s counter since it
-				// is the lower index.
-				configCounter[j]++
-				break
-			} else if j == i {
-				// serverConfigs[i] is equal to no
-				// other value seen before. It is
-				// unique so far.
-				configCounter[i] = 1
-				break
-			} // else invariants specified above are violated.
-		}
-	}
-
-	// We find the maximally occurring server config and check if
-	// there is quorum.
-	var configJSON serverConfig
-	maxOccurrence := 0
-	for i, count := range configCounter {
-		if maxOccurrence < count {
-			maxOccurrence = count
-			configJSON = serverConfigs[i]
-		}
-	}
-
-	// If quorum nodes don't agree.
-	if maxOccurrence < quorum {
-		return scv, errXLWriteQuorum
-	}
-
-	return configJSON, nil
-}
-
-// Write config contents into a temporary file on all nodes.
-func writeTmpConfigPeers(peers adminPeers, tmpFileName string, configBytes []byte) []error {
-	// For a single-node minio server setup.
-	if !globalIsDistXL {
-		err := peers[0].cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
-		return []error{err}
-	}
-
-	errs := make([]error, len(peers))
-
-	// Write config into temporary file on all nodes.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			errs[idx] = peer.cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Return bytes written and errors (if any) during writing
-	// temporary config file.
-	return errs
-}
-
-// Move config contents from the given temporary file onto config.json
-// on all nodes.
-func commitConfigPeers(peers adminPeers, tmpFileName string) []error {
-	// For a single-node minio server setup.
-	if !globalIsDistXL {
-		return []error{peers[0].cmdRunner.CommitConfig(tmpFileName)}
-	}
-
-	errs := make([]error, len(peers))
-
-	// Rename temporary config file into configDir/config.json on
-	// all nodes.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			errs[idx] = peer.cmdRunner.CommitConfig(tmpFileName)
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Return errors (if any) received during rename.
-	return errs
-}

cmd/admin-rpc-server.go

@@ -17,9 +17,7 @@
 package cmd
 
 import (
-	"context"
 	"path"
-	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
@@ -47,26 +45,29 @@ func (receiver *adminRPCReceiver) SignalService(args *SignalServiceArgs, reply *
 	return receiver.local.SignalService(args.Sig)
 }
 
-// ListLocksQuery - wraps ListLocks API's query values to send over RPC.
-type ListLocksQuery struct {
-	AuthArgs
-	Bucket   string
-	Prefix   string
-	Duration time.Duration
-}
-
-// ListLocks - lists locks held by requests handled by this server instance.
-func (receiver *adminRPCReceiver) ListLocks(args *ListLocksQuery, reply *[]VolumeLockInfo) (err error) {
-	*reply, err = receiver.local.ListLocks(args.Bucket, args.Prefix, args.Duration)
-	return err
-}
-
 // ServerInfo - returns the server info when object layer was initialized on this server.
 func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoData) (err error) {
 	*reply, err = receiver.local.ServerInfo()
 	return err
 }
 
+// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
+type StartProfilingArgs struct {
+	AuthArgs
+	Profiler string
+}
+
+// StartProfiling - starts profiling of this server
+func (receiver *adminRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
+	return receiver.local.StartProfiling(args.Profiler)
+}
+
+// DownloadProfilingData - stops and returns profiling data of this server
+func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) (err error) {
+	*reply, err = receiver.local.DownloadProfilingData()
+	return
+}
+
 // GetConfig - returns the config.json of this server.
 func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
 	*reply, err = receiver.local.GetConfig()
@@ -84,31 +85,6 @@ func (receiver *adminRPCReceiver) ReInitFormat(args *ReInitFormatArgs, reply *Vo
 	return receiver.local.ReInitFormat(args.DryRun)
 }
 
-// WriteConfigArgs - wraps the bytes to be written and temporary file name.
-type WriteConfigArgs struct {
-	AuthArgs
-	TmpFileName string
-	Buf         []byte
-}
-
-// WriteTmpConfig - writes the supplied config contents onto the
-// supplied temporary file.
-func (receiver *adminRPCReceiver) WriteTmpConfig(args *WriteConfigArgs, reply *VoidReply) error {
-	return receiver.local.WriteTmpConfig(args.TmpFileName, args.Buf)
-}
-
-// CommitConfigArgs - wraps the config file name that needs to be
-// committed into config.json on this node.
-type CommitConfigArgs struct {
-	AuthArgs
-	FileName string
-}
-
-// CommitConfig - Renames the temporary file into config.json on this node.
-func (receiver *adminRPCReceiver) CommitConfig(args *CommitConfigArgs, reply *VoidReply) error {
-	return receiver.local.CommitConfig(args.FileName)
-}
-
 // NewAdminRPCServer - returns new admin RPC server.
 func NewAdminRPCServer() (*xrpc.Server, error) {
 	rpcServer := xrpc.NewServer()
@@ -121,7 +97,7 @@ func NewAdminRPCServer() (*xrpc.Server, error) {
 // registerAdminRPCRouter - creates and registers Admin RPC server and its router.
 func registerAdminRPCRouter(router *mux.Router) {
 	rpcServer, err := NewAdminRPCServer()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server")
 	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
-	subrouter.Path(adminServiceSubPath).Handler(rpcServer)
+	subrouter.Path(adminServiceSubPath).HandlerFunc(httpTraceHdrs(rpcServer.ServeHTTP))
 }

cmd/admin-rpc_test.go

@@ -17,13 +17,8 @@
 package cmd
 
 import (
-	"encoding/json"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"reflect"
 	"testing"
 	"time"
 
@@ -99,31 +94,6 @@ func testAdminCmdRunnerReInitFormat(t *testing.T, client adminCmdRunner) {
 	}
 }
 
-func testAdminCmdRunnerListLocks(t *testing.T, client adminCmdRunner) {
-	tmpGlobalObjectAPI := globalObjectAPI
-	defer func() {
-		globalObjectAPI = tmpGlobalObjectAPI
-	}()
-
-	testCases := []struct {
-		objectAPI ObjectLayer
-		expectErr bool
-	}{
-		{&DummyObjectLayer{}, false},
-		{nil, true},
-	}
-
-	for i, testCase := range testCases {
-		globalObjectAPI = testCase.objectAPI
-		_, err := client.ListLocks("", "", time.Duration(0))
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
 func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
 	tmpGlobalBootTime := globalBootTime
 	tmpGlobalObjectAPI := globalObjectAPI
@@ -139,10 +109,8 @@ func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
 	}()
 
 	endpoints := new(EndpointList)
-	notificationSys, err := NewNotificationSys(globalServerConfig, *endpoints)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
+
+	notificationSys := NewNotificationSys(globalServerConfig, *endpoints)
 
 	testCases := []struct {
 		bootTime        time.Time
@@ -199,81 +167,6 @@ func testAdminCmdRunnerGetConfig(t *testing.T, client adminCmdRunner) {
 	}
 }
 
-func testAdminCmdRunnerWriteTmpConfig(t *testing.T, client adminCmdRunner) {
-	tmpConfigDir := configDir
-	defer func() {
-		configDir = tmpConfigDir
-	}()
-
-	tempDir, err := ioutil.TempDir("", ".AdminCmdRunnerWriteTmpConfig.")
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	defer os.RemoveAll(tempDir)
-	configDir = &ConfigDir{dir: tempDir}
-
-	testCases := []struct {
-		tmpFilename string
-		configBytes []byte
-		expectErr   bool
-	}{
-		{"config1.json", []byte(`{"version":"23","region":"us-west-1a"}`), false},
-		// Overwrite test.
-		{"config1.json", []byte(`{"version":"23","region":"us-west-1a","browser":"on"}`), false},
-		{"config2.json", []byte{}, false},
-		{"config3.json", nil, false},
-	}
-
-	for i, testCase := range testCases {
-		err := client.WriteTmpConfig(testCase.tmpFilename, testCase.configBytes)
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func testAdminCmdRunnerCommitConfig(t *testing.T, client adminCmdRunner) {
-	tmpConfigDir := configDir
-	defer func() {
-		configDir = tmpConfigDir
-	}()
-
-	tempDir, err := ioutil.TempDir("", ".AdminCmdRunnerCommitConfig.")
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	defer os.RemoveAll(tempDir)
-	configDir = &ConfigDir{dir: tempDir}
-	err = ioutil.WriteFile(filepath.Join(tempDir, "config.json"), []byte{}, os.ModePerm)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	err = client.WriteTmpConfig("config1.json", []byte(`{"version":"23","region":"us-west-1a"}`))
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	testCases := []struct {
-		tmpFilename string
-		expectErr   bool
-	}{
-		{"config1.json", false},
-		{"config2.json", true},
-	}
-
-	for i, testCase := range testCases {
-		err := client.CommitConfig(testCase.tmpFilename)
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
 func newAdminRPCHTTPServerClient(t *testing.T) (*httptest.Server, *AdminRPCClient, *serverConfig) {
 	rpcServer, err := NewAdminRPCServer()
 	if err != nil {
@@ -325,16 +218,6 @@ func TestAdminRPCClientReInitFormat(t *testing.T) {
 	testAdminCmdRunnerReInitFormat(t, rpcClient)
 }
 
-func TestAdminRPCClientListLocks(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerListLocks(t, rpcClient)
-}
-
 func TestAdminRPCClientServerInfo(t *testing.T) {
 	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
 	defer httpServer.Close()
@@ -354,260 +237,3 @@ func TestAdminRPCClientGetConfig(t *testing.T) {
 
 	testAdminCmdRunnerGetConfig(t, rpcClient)
 }
-
-func TestAdminRPCClientWriteTmpConfig(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerWriteTmpConfig(t, rpcClient)
-}
-
-func TestAdminRPCClientCommitConfig(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerCommitConfig(t, rpcClient)
-}
-
-var (
-	config1 = []byte(`{
-	"version": "13",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "us-east-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "debug"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	}
-}
-`)
-	// diff from config1 - amqp.Enable is True
-	config2 = []byte(`{
-	"version": "13",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "us-east-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "debug"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": true,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	}
-}
-`)
-)
-
-// TestGetValidServerConfig - test for getValidServerConfig.
-func TestGetValidServerConfig(t *testing.T) {
-	var c1, c2 serverConfig
-	err := json.Unmarshal(config1, &c1)
-	if err != nil {
-		t.Fatalf("json unmarshal of %s failed: %v", string(config1), err)
-	}
-
-	err = json.Unmarshal(config2, &c2)
-	if err != nil {
-		t.Fatalf("json unmarshal of %s failed: %v", string(config2), err)
-	}
-
-	// Valid config.
-	noErrs := []error{nil, nil, nil, nil}
-	serverConfigs := []serverConfig{c1, c2, c1, c1}
-	validConfig, err := getValidServerConfig(serverConfigs, noErrs)
-	if err != nil {
-		t.Errorf("Expected a valid config but received %v instead", err)
-	}
-
-	if !reflect.DeepEqual(validConfig, c1) {
-		t.Errorf("Expected valid config to be %v but received %v", config1, validConfig)
-	}
-
-	// Invalid config - no quorum.
-	serverConfigs = []serverConfig{c1, c2, c2, c1}
-	_, err = getValidServerConfig(serverConfigs, noErrs)
-	if err != errXLWriteQuorum {
-		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
-	}
-
-	// All errors
-	allErrs := []error{errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound}
-	serverConfigs = []serverConfig{{}, {}, {}, {}}
-	_, err = getValidServerConfig(serverConfigs, allErrs)
-	if err != errXLWriteQuorum {
-		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
-	}
-}

cmd/api-datatypes.go

@@ -23,6 +23,8 @@ import (
 const (
 	// Response request id.
 	responseRequestIDKey = "x-amz-request-id"
+	// Deployment id.
+	responseDeploymentIDKey = "x-minio-deployment-id"
 )
 
 // ObjectIdentifier carries key name for the object to delete.

cmd/api-errors_test.go

@@ -17,9 +17,11 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"testing"
 
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/hash"
 )
 
@@ -51,12 +53,11 @@ var toAPIErrorCodeTests = []struct {
 	{err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch},
 
 	// SSE-C errors
-	{err: errInsecureSSERequest, errCode: ErrInsecureSSECustomerRequest},
-	{err: errInvalidSSEAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
-	{err: errMissingSSEKey, errCode: ErrMissingSSECustomerKey},
-	{err: errInvalidSSEKey, errCode: ErrInvalidSSECustomerKey},
-	{err: errMissingSSEKeyMD5, errCode: ErrMissingSSECustomerKeyMD5},
-	{err: errSSEKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch},
+	{err: crypto.ErrInvalidCustomerAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
+	{err: crypto.ErrMissingCustomerKey, errCode: ErrMissingSSECustomerKey},
+	{err: crypto.ErrInvalidCustomerKey, errCode: ErrInvalidSSECustomerKey},
+	{err: crypto.ErrMissingCustomerKeyMD5, errCode: ErrMissingSSECustomerKeyMD5},
+	{err: crypto.ErrCustomerKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch},
 	{err: errObjectTampered, errCode: ErrObjectTampered},
 
 	{err: nil, errCode: ErrNone},
@@ -64,8 +65,9 @@ var toAPIErrorCodeTests = []struct {
 }
 
 func TestAPIErrCode(t *testing.T) {
+	ctx := context.Background()
 	for i, testCase := range toAPIErrorCodeTests {
-		errCode := toAPIErrorCode(testCase.err)
+		errCode := toAPIErrorCode(ctx, testCase.err)
 		if errCode != testCase.errCode {
 			t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)
 		}

cmd/api-headers.go

@@ -24,6 +24,8 @@ import (
 	"net/http"
 	"strconv"
 	"time"
+
+	"github.com/minio/minio/cmd/crypto"
 )
 
 // Returns a hexadecimal representation of time at the
@@ -34,8 +36,6 @@ func mustGetRequestID(t time.Time) string {
 
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
-	// Set unique request ID for each reply.
-	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
 	w.Header().Set("Server", globalServerUserAgent)
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
@@ -43,6 +43,9 @@ func setCommonHeaders(w http.ResponseWriter) {
 		w.Header().Set("X-Amz-Bucket-Region", region)
 	}
 	w.Header().Set("Accept-Ranges", "bytes")
+
+	// Remove sensitive information
+	crypto.RemoveSensitiveHeaders(w.Header())
 }
 
 // Encodes the response headers into XML format.
@@ -63,13 +66,10 @@ func encodeResponseJSON(response interface{}) []byte {
 }
 
 // Write object header
-func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) {
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec) (err error) {
 	// set common headers
 	setCommonHeaders(w)
 
-	// Set content length.
-	w.Header().Set("Content-Length", strconv.FormatInt(objInfo.Size, 10))
-
 	// Set last modified time.
 	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
 	w.Header().Set("Last-Modified", lastModified)
@@ -97,11 +97,34 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 		w.Header().Set(k, v)
 	}
 
-	// for providing ranged content
-	if contentRange != nil && contentRange.offsetBegin > -1 {
-		// Override content-length
-		w.Header().Set("Content-Length", strconv.FormatInt(contentRange.getLength(), 10))
-		w.Header().Set("Content-Range", contentRange.String())
-		w.WriteHeader(http.StatusPartialContent)
+	var totalObjectSize int64
+	switch {
+	case crypto.IsEncrypted(objInfo.UserDefined):
+		totalObjectSize, err = objInfo.DecryptedSize()
+		if err != nil {
+			return err
+		}
+	case objInfo.IsCompressed():
+		totalObjectSize = objInfo.GetActualSize()
+		if totalObjectSize < 0 {
+			return errInvalidDecompressedSize
+		}
+	default:
+		totalObjectSize = objInfo.Size
 	}
+
+	// for providing ranged content
+	start, rangeLen, err := rs.GetOffsetLength(totalObjectSize)
+	if err != nil {
+		return err
+	}
+
+	// Set content length.
+	w.Header().Set("Content-Length", strconv.FormatInt(rangeLen, 10))
+	if rs != nil {
+		contentRange := fmt.Sprintf("bytes %d-%d/%d", start, start+rangeLen-1, totalObjectSize)
+		w.Header().Set("Content-Range", contentRange)
+	}
+
+	return nil
 }

cmd/api-resources.go

@@ -22,59 +22,101 @@ import (
 )
 
 // Parse bucket url queries
-func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
-	prefix = values.Get("prefix")
-	marker = values.Get("marker")
-	delimiter = values.Get("delimiter")
+func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
 	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
 	} else {
 		maxkeys = maxObjectList
 	}
+
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse bucket url queries for ListObjects V2.
-func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string) {
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	// The continuation-token cannot be empty.
+	if val, ok := values["continuation-token"]; ok {
+		if len(val[0]) == 0 {
+			errCode = ErrIncorrectContinuationToken
+			return
+		}
+	}
+
+	if values.Get("max-keys") != "" {
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
+	} else {
+		maxkeys = maxObjectList
+	}
+
 	prefix = values.Get("prefix")
 	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
-	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
-	} else {
-		maxkeys = maxObjectList
-	}
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse bucket url queries for ?uploads
-func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
+func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	if values.Get("max-uploads") != "" {
+		var err error
+		if maxUploads, err = strconv.Atoi(values.Get("max-uploads")); err != nil {
+			errCode = ErrInvalidMaxUploads
+			return
+		}
+	} else {
+		maxUploads = maxUploadsList
+	}
+
 	prefix = values.Get("prefix")
 	keyMarker = values.Get("key-marker")
 	uploadIDMarker = values.Get("upload-id-marker")
 	delimiter = values.Get("delimiter")
-	if values.Get("max-uploads") != "" {
-		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
-	} else {
-		maxUploads = maxUploadsList
-	}
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse object url queries
-func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
-	uploadID = values.Get("uploadId")
-	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string, errCode APIErrorCode) {
+	var err error
+	errCode = ErrNone
+
 	if values.Get("max-parts") != "" {
-		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
+		if maxParts, err = strconv.Atoi(values.Get("max-parts")); err != nil {
+			errCode = ErrInvalidMaxParts
+			return
+		}
 	} else {
 		maxParts = maxPartsList
 	}
+
+	if values.Get("part-number-marker") != "" {
+		if partNumberMarker, err = strconv.Atoi(values.Get("part-number-marker")); err != nil {
+			errCode = ErrInvalidPartNumberMarker
+			return
+		}
+	}
+
+	uploadID = values.Get("uploadId")
 	encodingType = values.Get("encoding-type")
 	return
 }

cmd/api-resources_test.go

@@ -29,6 +29,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		fetchOwner                           bool
 		maxKeys                              int
 		encodingType                         string
+		errCode                              APIErrorCode
 	}{
 		{
 			values: url.Values{
@@ -47,6 +48,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      100,
 			encodingType: "gzip",
+			errCode:      ErrNone,
 		},
 		{
 			values: url.Values{
@@ -64,11 +66,34 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      1000,
 			encodingType: "gzip",
+			errCode:      ErrNone,
+		},
+		{
+			values: url.Values{
+				"prefix":             []string{"photos/"},
+				"continuation-token": []string{""},
+				"start-after":        []string{"start-after"},
+				"delimiter":          []string{"/"},
+				"fetch-owner":        []string{"true"},
+				"encoding-type":      []string{"gzip"},
+			},
+			prefix:       "",
+			token:        "",
+			startAfter:   "",
+			delimiter:    "",
+			fetchOwner:   false,
+			maxKeys:      0,
+			encodingType: "",
+			errCode:      ErrIncorrectContinuationToken,
 		},
 	}
 
 	for i, testCase := range testCases {
-		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType := getListObjectsV2Args(testCase.values)
+		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(testCase.values)
+
+		if errCode != testCase.errCode {
+			t.Errorf("Test %d: Expected error code:%d, got %d", i+1, testCase.errCode, errCode)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}
@@ -131,7 +156,10 @@ func TestListObjectsV1Resources(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		prefix, marker, delimiter, maxKeys, encodingType := getListObjectsV1Args(testCase.values)
+		prefix, marker, delimiter, maxKeys, encodingType, argsErr := getListObjectsV1Args(testCase.values)
+		if argsErr != ErrNone {
+			t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}
@@ -173,7 +201,10 @@ func TestGetObjectsResources(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		uploadID, partNumberMarker, maxParts, encodingType := getObjectResources(testCase.values)
+		uploadID, partNumberMarker, maxParts, encodingType, argsErr := getObjectResources(testCase.values)
+		if argsErr != ErrNone {
+			t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
+		}
 		if uploadID != testCase.uploadID {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.uploadID, uploadID)
 		}

cmd/api-response-multipart.go

@@ -16,7 +16,10 @@
 
 package cmd
 
-import "net/http"
+import (
+	"context"
+	"net/http"
+)
 
 // Represents additional fields necessary for ErrPartTooSmall S3 error.
 type completeMultipartAPIError struct {
@@ -42,9 +45,9 @@ type completeMultipartAPIError struct {
 // of this function.
 func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err PartTooSmall) {
 
-	apiError := getAPIError(toAPIErrorCode(err))
+	apiError := getAPIError(toAPIErrorCode(context.Background(), err))
 	// Generate complete multipart error response.
-	errorResponse := getAPIErrorResponse(apiError, r.URL.Path)
+	errorResponse := getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey))
 	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
 	encodedErrorResponse := encodeResponse(cmpErrResp)
 

cmd/api-response.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/pkg/handlers"
@@ -293,8 +294,10 @@ func getObjectLocation(r *http.Request, domain, bucket, object string) string {
 	}
 	// If domain is set then we need to use bucket DNS style.
 	if domain != "" {
-		u.Host = bucket + "." + domain
-		u.Path = path.Join(slashSeparator, object)
+		if strings.Contains(r.Host, domain) {
+			u.Host = bucket + "." + r.Host
+			u.Path = path.Join(slashSeparator, object)
+		}
 	}
 	return u.String()
 }
@@ -565,16 +568,23 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 }
 
 // writeErrorRespone writes error headers
-func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
+func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL, browser bool) {
 	switch errorCode {
 	case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set("Retry-After", "120")
+	case ErrAccessDenied:
+		if browser {
+			w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
+			w.WriteHeader(http.StatusTemporaryRedirect)
+			return
+		}
 	}
+
 	apiError := getAPIError(errorCode)
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
+	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
 	encodedErrorResponse := encodeResponse(errorResponse)
 	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
@@ -589,7 +599,7 @@ func writeErrorResponseHeadersOnly(w http.ResponseWriter, errorCode APIErrorCode
 func writeErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
 	apiError := getAPIError(errorCode)
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
+	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
@@ -605,7 +615,7 @@ func writeCustomErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode,
 		Code:      apiError.Code,
 		Message:   errBody,
 		Resource:  reqURL.Path,
-		RequestID: "3L137",
+		RequestID: w.Header().Get(responseRequestIDKey),
 		HostID:    "3L137",
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)

cmd/api-router.go

@@ -20,7 +20,6 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
 )
 
 // objectAPIHandler implements and provides http handlers for S3 API.
@@ -31,14 +30,6 @@ type objectAPIHandlers struct {
 
 // registerAPIRouter - registers S3 compatible APIs.
 func registerAPIRouter(router *mux.Router) {
-	var err error
-	var cacheConfig = globalServerConfig.GetCacheConfig()
-	if len(cacheConfig.Drives) > 0 {
-		// initialize the new disk cache objects.
-		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
-		logger.FatalIf(err, "Unable to initialize disk caching")
-	}
-
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
@@ -71,6 +62,8 @@ func registerAPIRouter(router *mux.Router) {
 		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
+		// SelectObjectContent
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
 		// GetObject
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
 		// CopyObject
@@ -108,7 +101,7 @@ func registerAPIRouter(router *mux.Router) {
 		// HeadBucket
 		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
 		// PostPolicy
-		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
+		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
 		// DeleteMultipleObjects
 		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
 		// DeleteBucketPolicy

cmd/auth-handler.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/subtle"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
@@ -27,8 +28,11 @@ import (
 	"net/http"
 	"strings"
 
+	jwtgo "github.com/dgrijalva/jwt-go"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -86,6 +90,7 @@ const (
 	authTypeSigned
 	authTypeSignedV2
 	authTypeJWT
+	authTypeSTS
 )
 
 // Get request authentication type.
@@ -104,6 +109,8 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypeJWT
 	} else if isRequestPostPolicySignatureV4(r) {
 		return authTypePostPolicy
+	} else if _, ok := r.URL.Query()["Action"]; ok {
+		return authTypeSTS
 	} else if _, ok := r.Header["Authorization"]; !ok {
 		return authTypeAnonymous
 	}
@@ -112,43 +119,140 @@ func getRequestAuthType(r *http.Request) authType {
 
 // checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
 // It does not accept presigned or JWT or anonymous requests.
-func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
+func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region string) APIErrorCode {
 	s3Err := ErrAccessDenied
-	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok && getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) { // we only support V4 (no presign) with auth. body
-		s3Err = isReqAuthenticated(r, region)
+	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok &&
+		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
+		// We only support admin credentials to access admin APIs.
+
+		var owner bool
+		_, owner, s3Err = getReqAccessKeyV4(r, region)
+		if s3Err != ErrNone {
+			return s3Err
+		}
+
+		if !owner {
+			return ErrAccessDenied
+		}
+
+		// we only support V4 (no presign) with auth body
+		s3Err = isReqAuthenticated(ctx, r, region)
 	}
 	if s3Err != ErrNone {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		ctx := logger.SetReqInfo(ctx, reqInfo)
 		logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description))
 	}
 	return s3Err
 }
 
-func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) APIErrorCode {
-	isOwner := true
-	accountName := globalServerConfig.GetCredential().AccessKey
+// Fetch the security token set by the client.
+func getSessionToken(r *http.Request) (token string) {
+	token = r.Header.Get("X-Amz-Security-Token")
+	if token != "" {
+		return token
+	}
+	return r.URL.Query().Get("X-Amz-Security-Token")
+}
 
+// Fetch claims in the security token returned by the client, doesn't return
+// errors - upon errors the returned claims map will be empty.
+func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
+	claims, _ := getClaimsFromToken(r)
+	return claims
+}
+
+// Fetch claims in the security token returned by the client.
+func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
+	claims := make(map[string]interface{})
+	token := getSessionToken(r)
+	if token == "" {
+		return claims, nil
+	}
+	stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
+		// JWT token for x-amz-security-token is signed with admin
+		// secret key, temporary credentials become invalid if
+		// server admin credentials change. This is done to ensure
+		// that clients cannot decode the token using the temp
+		// secret keys and generate an entirely new claim by essentially
+		// hijacking the policies. We need to make sure that this is
+		// based an admin credential such that token cannot be decoded
+		// on the client side and is treated like an opaque value.
+		return []byte(globalServerConfig.GetCredential().SecretKey), nil
+	}
+	p := &jwtgo.Parser{
+		ValidMethods: []string{
+			jwtgo.SigningMethodHS256.Alg(),
+			jwtgo.SigningMethodHS512.Alg(),
+		},
+	}
+	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
+	if err != nil {
+		return nil, err
+	}
+	if !jtoken.Valid {
+		return nil, errAuthentication
+	}
+	v, ok := claims["accessKey"]
+	if !ok {
+		return nil, errInvalidAccessKeyID
+	}
+	if _, ok = v.(string); !ok {
+		return nil, errInvalidAccessKeyID
+	}
+	return claims, nil
+}
+
+// Fetch claims in the security token returned by the client and validate the token.
+func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
+	token := getSessionToken(r)
+	if token != "" && cred.AccessKey == "" {
+		return nil, ErrNoAccessKey
+	}
+	if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+		return nil, ErrInvalidToken
+	}
+	claims, err := getClaimsFromToken(r)
+	if err != nil {
+		return nil, toAPIErrorCode(context.Background(), err)
+	}
+	return claims, ErrNone
+}
+
+// Check request auth type verifies the incoming http request
+// - validates the request signature
+// - validates the policy action if anonymous tests bucket policies if any,
+//   for authenticated requests validates IAM policies.
+// returns APIErrorCode if any to be replied to the client.
+func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
+	var cred auth.Credentials
+	var owner bool
 	switch getRequestAuthType(r) {
-	case authTypeUnknown:
+	case authTypeUnknown, authTypeStreamingSigned:
 		return ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
-		if errorCode := isReqAuthenticatedV2(r); errorCode != ErrNone {
-			return errorCode
+		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
+			return s3Err
 		}
+		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
 		region := globalServerConfig.GetRegion()
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
 		}
-
-		if errorCode := isReqAuthenticated(r, region); errorCode != ErrNone {
-			return errorCode
+		if s3Err = isReqAuthenticated(ctx, r, region); s3Err != ErrNone {
+			return s3Err
 		}
-	default:
-		isOwner = false
-		accountName = ""
+		cred, owner, s3Err = getReqAccessKeyV4(r, region)
+	}
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	claims, s3Err := checkClaimsFromToken(r, cred)
+	if s3Err != ErrNone {
+		return s3Err
 	}
 
 	// LocationConstraint is valid only for CreateBucketAction.
@@ -174,17 +278,31 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
 
-	if globalPolicySys.IsAllowed(policy.Args{
-		AccountName:     accountName,
-		Action:          action,
+	if cred.AccessKey == "" {
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          action,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, locationConstraint),
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
+		Action:          iampolicy.Action(action),
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, locationConstraint),
-		IsOwner:         isOwner,
+		ConditionValues: getConditionValues(r, ""),
 		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
 	}) {
 		return ErrNone
 	}
-
 	return ErrAccessDenied
 }
 
@@ -209,7 +327,7 @@ func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode)
 }
 
 // Verify if request has valid AWS Signature Version '4'.
-func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
+func isReqAuthenticated(ctx context.Context, r *http.Request, region string) (s3Error APIErrorCode) {
 	if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
 		return errCode
 	}
@@ -244,9 +362,9 @@ func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
 
 	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
 	// The verification happens implicit during reading.
-	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256))
+	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1)
 	if err != nil {
-		return toAPIErrorCode(err)
+		return toAPIErrorCode(ctx, err)
 	}
 	r.Body = ioutil.NopCloser(reader)
 	return ErrNone
@@ -288,12 +406,67 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	} else if aType == authTypeJWT {
 		// Validate Authorization header if its valid for JWT request.
-		if !isHTTPRequestValid(r) {
+		if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
 			w.WriteHeader(http.StatusUnauthorized)
 			return
 		}
 		a.handler.ServeHTTP(w, r)
 		return
+	} else if aType == authTypeSTS {
+		a.handler.ServeHTTP(w, r)
+		return
 	}
-	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL)
+	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL, guessIsBrowserReq(r))
+}
+
+// isPutAllowed - check if PUT operation is allowed on the resource, this
+// call verifies bucket policies and IAM policies, supports multi user
+// checks etc.
+func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request) (s3Err APIErrorCode) {
+	var cred auth.Credentials
+	var owner bool
+	switch atype {
+	case authTypeUnknown:
+		return ErrAccessDenied
+	case authTypeSignedV2, authTypePresignedV2:
+		cred, owner, s3Err = getReqAccessKeyV2(r)
+	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
+		region := globalServerConfig.GetRegion()
+		cred, owner, s3Err = getReqAccessKeyV4(r, region)
+	}
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	claims, s3Err := checkClaimsFromToken(r, cred)
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	if cred.AccessKey == "" {
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          policy.PutObjectAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
+		Action:          policy.PutObjectAction,
+		BucketName:      bucketName,
+		ConditionValues: getConditionValues(r, ""),
+		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
+	}) {
+		return ErrNone
+	}
+	return ErrAccessDenied
 }

cmd/auth-handler_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -344,11 +345,14 @@ func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int6
 
 // Tests is requested authenticated function, tests replies for s3 errors.
 func TestIsReqAuthenticated(t *testing.T) {
-	path, err := newTestConfig(globalMinioDefaultRegion)
+	objLayer, fsDir, err := prepareFS()
 	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
@@ -374,21 +378,26 @@ func TestIsReqAuthenticated(t *testing.T) {
 		{mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrNone},
 	}
 
+	ctx := context.Background()
 	// Validates all testcases.
 	for i, testCase := range testCases {
-		if s3Error := isReqAuthenticated(testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
-			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(err) != testCase.s3Error {
-				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %d)", i, testCase.s3Error, s3Error, toAPIErrorCode(err))
+		if s3Error := isReqAuthenticated(ctx, testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
+			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
+				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %d)", i, testCase.s3Error, s3Error, toAPIErrorCode(ctx, err))
 			}
 		}
 	}
 }
 func TestCheckAdminRequestAuthType(t *testing.T) {
-	path, err := newTestConfig(globalMinioDefaultRegion)
+	objLayer, fsDir, err := prepareFS()
 	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
@@ -406,8 +415,9 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		{Request: mustNewPresignedV2Request("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 		{Request: mustNewPresignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 	}
+	ctx := context.Background()
 	for i, testCase := range testCases {
-		if s3Error := checkAdminRequestAuthType(testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
+		if s3Error := checkAdminRequestAuthType(ctx, testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}

cmd/benchmark-utils_test.go

@@ -22,7 +22,6 @@ import (
 	"io/ioutil"
 	"math"
 	"math/rand"
-	"os"
 	"strconv"
 	"testing"
 
@@ -62,7 +61,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < b.N; i++ {
 		// insert the object.
 		objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -98,7 +97,8 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
 	metadata := make(map[string]string)
-	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata)
+	opts := ObjectOptions{}
+	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata, opts)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -123,7 +123,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			md5hex = getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
 			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
-				mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex))
+				mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), opts)
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -138,12 +138,6 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 
 // creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
 func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
 	if err != nil {
@@ -151,18 +145,13 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
 	}
 	// cleaning up the backend by removing all the directories and files created on function return.
 	defer removeRoots(disks)
+
 	// uses *testing.B and the object Layer to run the benchmark.
 	runPutObjectPartBenchmark(b, objLayer, objSize)
 }
 
 // creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
 func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
 	if err != nil {
@@ -170,18 +159,13 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
 	}
 	// cleaning up the backend by removing all the directories and files created on function return.
 	defer removeRoots(disks)
+
 	// uses *testing.B and the object Layer to run the benchmark.
 	runPutObjectBenchmark(b, objLayer, objSize)
 }
 
 // creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
 func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
 	if err != nil {
@@ -189,6 +173,7 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
 	}
 	// cleaning up the backend by removing all the directories and files created on function return.
 	defer removeRoots(disks)
+
 	// uses *testing.B and the object Layer to run the benchmark.
 	runPutObjectBenchmarkParallel(b, objLayer, objSize)
 }
@@ -196,16 +181,10 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
 // Benchmark utility functions for ObjectLayer.GetObject().
 // Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
 func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -225,7 +204,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -240,7 +219,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		var buffer = new(bytes.Buffer)
-		err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "")
+		err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "", ObjectOptions{})
 		if err != nil {
 			b.Error(err)
 		}
@@ -269,12 +248,6 @@ func generateBytesData(size int) []byte {
 
 // creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
 func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
 	if err != nil {
@@ -282,18 +255,13 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
 	}
 	// cleaning up the backend by removing all the directories and files created.
 	defer removeRoots(disks)
+
 	//  uses *testing.B and the object Layer to run the benchmark.
 	runGetObjectBenchmark(b, objLayer, objSize)
 }
 
 // creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
 func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
 	if err != nil {
@@ -301,6 +269,7 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
 	}
 	// cleaning up the backend by removing all the directories and files created.
 	defer removeRoots(disks)
+
 	//  uses *testing.B and the object Layer to run the benchmark.
 	runGetObjectBenchmarkParallel(b, objLayer, objSize)
 }
@@ -308,16 +277,10 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
 // Parallel benchmark utility functions for ObjectLayer.PutObject().
 // Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
 func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -341,7 +304,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		for pb.Next() {
 			// insert the object.
 			objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-				mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+				mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -359,16 +322,10 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 // Parallel benchmark utility functions for ObjectLayer.GetObject().
 // Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
 func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		b.Fatalf("Unable to initialize config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -387,7 +344,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -403,7 +360,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	b.RunParallel(func(pb *testing.PB) {
 		i := 0
 		for pb.Next() {
-			err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "")
+			err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "", ObjectOptions{})
 			if err != nil {
 				b.Error(err)
 			}

cmd/bitrot.go

@@ -0,0 +1,194 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"hash"
+
+	"github.com/minio/highwayhash"
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"golang.org/x/crypto/blake2b"
+)
+
+// magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key.
+var magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0")
+
+// BitrotAlgorithm specifies a algorithm used for bitrot protection.
+type BitrotAlgorithm uint
+
+const (
+	// SHA256 represents the SHA-256 hash function
+	SHA256 BitrotAlgorithm = 1 + iota
+	// HighwayHash256 represents the HighwayHash-256 hash function
+	HighwayHash256
+	// BLAKE2b512 represents the BLAKE2b-512 hash function
+	BLAKE2b512
+)
+
+// DefaultBitrotAlgorithm is the default algorithm used for bitrot protection.
+const (
+	DefaultBitrotAlgorithm = HighwayHash256
+)
+
+var bitrotAlgorithms = map[BitrotAlgorithm]string{
+	SHA256:         "sha256",
+	BLAKE2b512:     "blake2b",
+	HighwayHash256: "highwayhash256",
+}
+
+// New returns a new hash.Hash calculating the given bitrot algorithm.
+func (a BitrotAlgorithm) New() hash.Hash {
+	switch a {
+	case SHA256:
+		return sha256.New()
+	case BLAKE2b512:
+		b2, _ := blake2b.New512(nil) // New512 never returns an error if the key is nil
+		return b2
+	case HighwayHash256:
+		hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
+		return hh
+	default:
+		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
+		return nil
+	}
+}
+
+// Available reports whether the given algorithm is available.
+func (a BitrotAlgorithm) Available() bool {
+	_, ok := bitrotAlgorithms[a]
+	return ok
+}
+
+// String returns the string identifier for a given bitrot algorithm.
+// If the algorithm is not supported String panics.
+func (a BitrotAlgorithm) String() string {
+	name, ok := bitrotAlgorithms[a]
+	if !ok {
+		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
+	}
+	return name
+}
+
+// NewBitrotVerifier returns a new BitrotVerifier implementing the given algorithm.
+func NewBitrotVerifier(algorithm BitrotAlgorithm, checksum []byte) *BitrotVerifier {
+	return &BitrotVerifier{algorithm, checksum}
+}
+
+// BitrotVerifier can be used to verify protected data.
+type BitrotVerifier struct {
+	algorithm BitrotAlgorithm
+	sum       []byte
+}
+
+// BitrotAlgorithmFromString returns a bitrot algorithm from the given string representation.
+// It returns 0 if the string representation does not match any supported algorithm.
+// The zero value of a bitrot algorithm is never supported.
+func BitrotAlgorithmFromString(s string) (a BitrotAlgorithm) {
+	for alg, name := range bitrotAlgorithms {
+		if name == s {
+			return alg
+		}
+	}
+	return
+}
+
+// To read bit-rot verified data.
+type bitrotReader struct {
+	disk      StorageAPI
+	volume    string
+	filePath  string
+	verifier  *BitrotVerifier // Holds the bit-rot info
+	endOffset int64           // Affects the length of data requested in disk.ReadFile depending on Read()'s offset
+	buf       []byte          // Holds bit-rot verified data
+}
+
+// newBitrotReader returns bitrotReader.
+// Note that the buffer is allocated later in Read(). This is because we will know the buffer length only
+// during the bitrotReader.Read(). Depending on when parallelReader fails-over, the buffer length can be different.
+func newBitrotReader(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm, endOffset int64, sum []byte) *bitrotReader {
+	return &bitrotReader{
+		disk:      disk,
+		volume:    volume,
+		filePath:  filePath,
+		verifier:  &BitrotVerifier{algo, sum},
+		endOffset: endOffset,
+		buf:       nil,
+	}
+}
+
+// ReadChunk returns requested data.
+func (b *bitrotReader) ReadChunk(offset int64, length int64) ([]byte, error) {
+	if b.buf == nil {
+		b.buf = make([]byte, b.endOffset-offset)
+		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
+			ctx := context.Background()
+			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
+			logger.LogIf(ctx, err)
+			return nil, err
+		}
+	}
+	if int64(len(b.buf)) < length {
+		logger.LogIf(context.Background(), errLessData)
+		return nil, errLessData
+	}
+	retBuf := b.buf[:length]
+	b.buf = b.buf[length:]
+	return retBuf, nil
+}
+
+// To calculate the bit-rot of the written data.
+type bitrotWriter struct {
+	disk     StorageAPI
+	volume   string
+	filePath string
+	h        hash.Hash
+}
+
+// newBitrotWriter returns bitrotWriter.
+func newBitrotWriter(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm) *bitrotWriter {
+	return &bitrotWriter{
+		disk:     disk,
+		volume:   volume,
+		filePath: filePath,
+		h:        algo.New(),
+	}
+}
+
+// Append appends the data and while calculating the hash.
+func (b *bitrotWriter) Append(buf []byte) error {
+	n, err := b.h.Write(buf)
+	if err != nil {
+		return err
+	}
+	if n != len(buf) {
+		logger.LogIf(context.Background(), errUnexpected)
+		return errUnexpected
+	}
+	if err = b.disk.AppendFile(b.volume, b.filePath, buf); err != nil {
+		logger.LogIf(context.Background(), err)
+		return err
+	}
+	return nil
+}
+
+// Sum returns bit-rot sum.
+func (b *bitrotWriter) Sum() []byte {
+	return b.h.Sum(nil)
+}

cmd/bitrot_test.go

@@ -0,0 +1,71 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"testing"
+)
+
+func TestBitrotReaderWriter(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	volume := "testvol"
+	filePath := "testfile"
+
+	disk, err := newPosix(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	disk.MakeVol(volume)
+
+	writer := newBitrotWriter(disk, volume, filePath, HighwayHash256)
+
+	err = writer.Append([]byte("aaaaaaaaa"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = writer.Append([]byte("a"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = writer.Append([]byte("aaaaaaaaaa"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = writer.Append([]byte("aaaaa"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = writer.Append([]byte("aaaaaaaaaa"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	reader := newBitrotReader(disk, volume, filePath, HighwayHash256, 35, writer.Sum())
+
+	if _, err = reader.ReadChunk(0, 35); err != nil {
+		log.Fatal(err)
+	}
+}

cmd/bool-flag.go

@@ -56,11 +56,12 @@ func (bf *BoolFlag) UnmarshalJSON(data []byte) (err error) {
 
 // ParseBoolFlag - parses string into BoolFlag.
 func ParseBoolFlag(s string) (bf BoolFlag, err error) {
-	if s == "on" {
+	switch s {
+	case "on":
 		bf = true
-	} else if s == "off" {
+	case "off":
 		bf = false
-	} else {
+	default:
 		err = fmt.Errorf("invalid value ‘%s’ for BoolFlag", s)
 	}
 

cmd/bucket-handlers-listobjects.go

@@ -20,6 +20,9 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/minio/cmd/logger"
+
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -54,37 +57,38 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 // NOTE: It is recommended that this API to be used for application development.
 // Minio continues to support ListObjectsV1 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListObjectsV2")
+	ctx := newContext(r, w, "ListObjectsV2")
+
+	defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+	urlValues := r.URL.Query()
 
-	// In ListObjectsV2 'continuation-token' is the marker.
-	marker := token
-	// Check if 'continuation-token' is empty.
-	if token == "" {
-		// Then we need to use 'start-after' as marker instead.
-		marker = startAfter
+	// Extract all the listObjectsV2 query params to their native values.
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
+
+	if errCode != ErrNone {
+		writeErrorResponse(w, errCode, r.URL, guessIsBrowserReq(r))
+		return
 	}
 
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+	if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	listObjectsV2 := objectAPI.ListObjectsV2
@@ -93,18 +97,29 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
-	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, fetchOwner, startAfter)
+	// marshaled into S3 compatible XML header.
+	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	for i := range listObjectsV2Info.Objects {
-		if listObjectsV2Info.Objects[i].IsEncrypted() {
+		var actualSize int64
+		if listObjectsV2Info.Objects[i].IsCompressed() {
+			// Read the decompressed size from the meta.json.
+			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
+			if actualSize < 0 {
+				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
+				return
+			}
+			// Set the info.Size to the actualSize.
+			listObjectsV2Info.Objects[i].Size = actualSize
+		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
+			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
 			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
@@ -124,33 +139,39 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 // criteria to return a subset of the objects in a bucket.
 //
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListObjectsV1")
+	ctx := newContext(r, w, "ListObjectsV1")
+
+	defer logger.AuditLog(w, r, "ListObjectsV1", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Extract all the litsObjectsV1 query params to their native values.
-	prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
+	prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
 	// does not throw an error.
 	if maxKeys < 0 {
-		writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
+		writeErrorResponse(w, ErrInvalidMaxKeys, r.URL, guessIsBrowserReq(r))
 		return
 	} // Validate all the query params before beginning to serve the request.
 	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	listObjects := objectAPI.ListObjects
@@ -159,23 +180,33 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
+	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	for i := range listObjectsInfo.Objects {
-		if listObjectsInfo.Objects[i].IsEncrypted() {
+		var actualSize int64
+		if listObjectsInfo.Objects[i].IsCompressed() {
+			// Read the decompressed size from the meta.json.
+			actualSize = listObjectsInfo.Objects[i].GetActualSize()
+			if actualSize < 0 {
+				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
+				return
+			}
+			// Set the info.Size to the actualSize.
+			listObjectsInfo.Objects[i].Size = actualSize
+		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
+			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
 			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
 	}
-
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
 
 	// Write success response.

cmd/bucket-handlers.go

@@ -17,8 +17,10 @@
 package cmd
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/xml"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -26,47 +28,89 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"sync"
 
 	"github.com/gorilla/mux"
+
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+// Check if there are buckets on server without corresponding entry in etcd backend and
+// make entries. Here is the general flow
+// - Range over all the available buckets
+// - Check if a bucket has an entry in etcd backend
+// -- If no, make an entry
+// -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
+// -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
+func initFederatorBackend(objLayer ObjectLayer) {
+	b, err := objLayer.ListBuckets(context.Background())
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return
+	}
+
+	g := errgroup.WithNErrs(len(b))
+	for index := range b {
+		index := index
+		g.Go(func() error {
+			r, gerr := globalDNSConfig.Get(b[index].Name)
+			if gerr != nil {
+				if gerr == dns.ErrNoEntriesFound {
+					return globalDNSConfig.Put(b[index].Name)
+				}
+				return gerr
+			}
+			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				// There is already an entry for this bucket, with all IP addresses different. This indicates a bucket name collision. Log an error and continue.
+				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", b[index].Name, globalDomainIPs.ToSlice())
+			}
+			return nil
+		}, index)
+	}
+
+	for _, err := range g.Wait() {
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			return
+		}
+	}
+}
+
 // GetBucketLocationHandler - GET Bucket location.
 // -------------------------
 // This operation returns bucket location.
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketLocation")
+	ctx := newContext(r, w, "GetBucketLocation")
+
+	defer logger.AuditLog(w, r, "GetBucketLocation", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetRLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-	defer bucketLock.RUnlock()
 	getBucketInfo := objectAPI.GetBucketInfo
 	if api.CacheAPI() != nil {
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -93,38 +137,44 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 // uploads in the response.
 //
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListMultipartUploads")
+	ctx := newContext(r, w, "ListMultipartUploads")
+
+	defer logger.AuditLog(w, r, "ListMultipartUploads", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, s3Error := getBucketMultipartResources(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		return
+	}
 	if maxUploads < 0 {
-		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL)
+		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	if keyMarker != "" {
 		// Marker not common with prefix is not implemented.
 		if !hasPrefix(keyMarker, prefix) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL)
+			writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// generate response
@@ -140,11 +190,13 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 // This implementation of the GET operation returns a list of all buckets
 // owned by the authenticated sender of the request.
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListBuckets")
+	ctx := newContext(r, w, "ListBuckets")
+
+	defer logger.AuditLog(w, r, "ListBuckets", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	listBuckets := objectAPI.ListBuckets
@@ -154,15 +206,36 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
-
-	// Invoke the list buckets.
-	bucketsInfo, err := listBuckets(ctx)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
+	// If etcd, dns federation configured list buckets from etcd.
+	var bucketsInfo []BucketInfo
+	if globalDNSConfig != nil {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil && err != dns.ErrNoEntriesFound {
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+		bucketSet := set.NewStringSet()
+		for _, dnsRecord := range dnsBuckets {
+			if bucketSet.Contains(dnsRecord.Key) {
+				continue
+			}
+			bucketsInfo = append(bucketsInfo, BucketInfo{
+				Name:    strings.Trim(dnsRecord.Key, slashSeparator),
+				Created: dnsRecord.CreationDate,
+			})
+			bucketSet.Add(dnsRecord.Key)
+		}
+	} else {
+		// Invoke the list buckets.
+		var err error
+		bucketsInfo, err = listBuckets(ctx)
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 
 	// Generate response.
@@ -175,14 +248,16 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 // DeleteMultipleObjectsHandler - deletes multiple objects.
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteMultipleObjects")
+	ctx := newContext(r, w, "DeleteMultipleObjects")
+
+	defer logger.AuditLog(w, r, "DeleteMultipleObjects", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -191,7 +266,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		// In the event access is denied, a 200 response should still be returned
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 		if s3Error != ErrAccessDenied {
-			writeErrorResponse(w, s3Error, r.URL)
+			writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -199,24 +274,30 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Content-Md5 is requied should be set
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(w, ErrMissingContentMD5, r.URL)
+		writeErrorResponse(w, ErrMissingContentMD5, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allocate incoming content length bytes.
-	deleteXMLBytes := make([]byte, r.ContentLength)
+	var deleteXMLBytes []byte
+	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
+	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
+		deleteXMLBytes = make([]byte, maxBodySize)
+	} else {
+		deleteXMLBytes = make([]byte, r.ContentLength)
+	}
 
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrInternalError, r.URL)
+		writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -224,7 +305,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedXML, r.URL)
+		writeErrorResponse(w, ErrMalformedXML, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -232,38 +313,28 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	if globalWORMEnabled {
 		// Not required to check whether given objects exist or not, because
 		// DeleteMultipleObject is always successful irrespective of object existence.
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+		writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	var wg = &sync.WaitGroup{} // Allocate a new wait group.
-	var dErrs = make([]error, len(deleteObjects.Objects))
-
-	// Delete all requested objects in parallel.
-	for index, object := range deleteObjects.Objects {
-		wg.Add(1)
-		go func(i int, obj ObjectIdentifier) {
-			defer wg.Done()
-			// If the request is denied access, each item
-			// should be marked as 'AccessDenied'
-			if s3Error == ErrAccessDenied {
-				dErrs[i] = PrefixAccessDenied{
-					Bucket: bucket,
-					Object: obj.ObjectName,
-				}
-				return
-			}
-			deleteObject := objectAPI.DeleteObject
-			if api.CacheAPI() != nil {
-				deleteObject = api.CacheAPI().DeleteObject
-			}
-			dErr := deleteObject(ctx, bucket, obj.ObjectName)
-			if dErr != nil {
-				dErrs[i] = dErr
-			}
-		}(index, object)
+	deleteObject := objectAPI.DeleteObject
+	if api.CacheAPI() != nil {
+		deleteObject = api.CacheAPI().DeleteObject
+	}
+
+	var dErrs = make([]error, len(deleteObjects.Objects))
+	for index, object := range deleteObjects.Objects {
+		// If the request is denied access, each item
+		// should be marked as 'AccessDenied'
+		if s3Error == ErrAccessDenied {
+			dErrs[index] = PrefixAccessDenied{
+				Bucket: bucket,
+				Object: object.ObjectName,
+			}
+			continue
+		}
+		dErrs[index] = deleteObject(ctx, bucket, object.ObjectName)
 	}
-	wg.Wait()
 
 	// Collect deleted objects and errors if any.
 	var deletedObjects []ObjectIdentifier
@@ -283,8 +354,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 		// Error during delete should be collected separately.
 		deleteErrors = append(deleteErrors, DeleteError{
-			Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
-			Message: errorCodeResponse[toAPIErrorCode(err)].Description,
+			Code:    errorCodeResponse[toAPIErrorCode(ctx, err)].Code,
+			Message: errorCodeResponse[toAPIErrorCode(ctx, err)].Description,
 			Key:     object.ObjectName,
 		})
 	}
@@ -298,7 +369,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Get host and port from Request.RemoteAddr failing which
 	// fill them with empty strings.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -311,10 +382,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			Object: ObjectInfo{
 				Name: dobj.ObjectName,
 			},
-			ReqParams: extractReqParams(r),
-			UserAgent: r.UserAgent(),
-			Host:      host,
-			Port:      port,
+			ReqParams:    extractReqParams(r),
+			RespElements: extractRespElements(w),
+			UserAgent:    r.UserAgent(),
+			Host:         host,
+			Port:         port,
 		})
 	}
 }
@@ -323,11 +395,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 // ----------
 // This implementation of the PUT operation creates a new bucket for authenticated request
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucket")
+	ctx := newContext(r, w, "PutBucket")
+
+	defer logger.AuditLog(w, r, "PutBucket", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -335,35 +409,56 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Validate if location sent by the client is valid, reject
 	// requests which do not follow valid region requirements.
 	if !isValidLocation(location) {
-		writeErrorResponse(w, ErrInvalidRegion, r.URL)
+		writeErrorResponse(w, ErrInvalidRegion, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	if globalDNSConfig != nil {
+		if _, err := globalDNSConfig.Get(bucket); err != nil {
+			if err == dns.ErrNoEntriesFound {
+				// Proceed to creating a bucket.
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+					return
+				}
+				if err = globalDNSConfig.Put(bucket); err != nil {
+					objectAPI.DeleteBucket(ctx, bucket)
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+					return
+				}
+
+				// Make sure to add Location information here only for bucket
+				w.Header().Set("Location", getObjectLocation(r, globalDomainName, bucket, ""))
+
+				writeSuccessResponseHeadersOnly(w)
+				return
+			}
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+
+		}
+		writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL, guessIsBrowserReq(r))
 		return
 	}
-	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -378,11 +473,13 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 // This implementation of the POST operation handles object creation with a specified
 // signature policy in multipart/form-data
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PostPolicyBucket")
+	ctx := newContext(r, w, "PostPolicyBucket")
+
+	defer logger.AuditLog(w, r, "PostPolicyBucket", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -391,17 +488,17 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
 	if err != nil {
-		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		writeErrorResponse(w, ErrInvalidRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// Make sure that the URL  does not contain object name.
 	if bucket != filepath.Clean(resource[1:]) {
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+		writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -410,7 +507,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	reader, err := r.MultipartReader()
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -418,7 +515,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	form, err := reader.ReadForm(maxFormMemory)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -429,13 +526,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if file is provided, error out otherwise.
 	if fileBody == nil {
-		writeErrorResponse(w, ErrPOSTFileRequired, r.URL)
+		writeErrorResponse(w, ErrPOSTFileRequired, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -457,7 +554,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if successRedirect != "" {
 		redirectURL, err = url.Parse(successRedirect)
 		if err != nil {
-			writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+			writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -465,25 +562,25 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Verify policy signature.
 	apiErr := doesPolicySignatureMatch(formValues)
 	if apiErr != ErrNone {
-		writeErrorResponse(w, apiErr, r.URL)
+		writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Make sure formValues adhere to policy restrictions.
 	if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
-		writeErrorResponse(w, apiErr, r.URL)
+		writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -492,56 +589,62 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	lengthRange := postPolicyForm.Conditions.ContentLengthRange
 	if lengthRange.Valid {
 		if fileSize < lengthRange.Min {
-			writeErrorResponse(w, toAPIErrorCode(errDataTooSmall), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
-			writeErrorResponse(w, toAPIErrorCode(errDataTooLarge), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	// Extract metadata to be saved from received Form.
-	metadata, err := extractMetadataFromHeader(ctx, formValues)
+	metadata := make(map[string]string)
+	err = extractMetadataFromMap(ctx, formValues, metadata)
 	if err != nil {
-		writeErrorResponse(w, ErrInternalError, r.URL)
+		writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
+	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
+	rawReader := hashReader
+	pReader := NewPutObjReader(rawReader, nil, nil)
+	var objectEncryptionKey []byte
 	if objectAPI.IsEncryptionSupported() {
-		if hasSSECustomerHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C requests
+		if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C and SSE-S3 requests
 			var reader io.Reader
 			var key []byte
-			key, err = ParseSSECustomerHeader(formValues)
-			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-				return
+			if crypto.SSEC.IsRequested(formValues) {
+				key, err = ParseSSECustomerHeader(formValues)
+				if err != nil {
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+					return
+				}
 			}
-			reader, err = newEncryptReader(hashReader, key, metadata)
+			reader, objectEncryptionKey, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			info := ObjectInfo{Size: fileSize}
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "") // do not try to verify encrypted content
+			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
+			pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
 		}
 	}
 
-	objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata)
+	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, metadata, ObjectOptions{})
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -550,20 +653,21 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	w.Header().Set("Location", location)
 
 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedPost,
-		BucketName: objInfo.Bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedPost,
+		BucketName:   objInfo.Bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 
 	if successRedirect != "" {
@@ -597,7 +701,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 // have permission to access it. Otherwise, the operation might
 // return responses such as 404 Not Found and 403 Forbidden.
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "HeadBucket")
+	ctx := newContext(r, w, "HeadBucket")
+
+	defer logger.AuditLog(w, r, "HeadBucket", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -618,7 +724,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
+		writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
 		return
 	}
 
@@ -627,19 +733,21 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 
 // DeleteBucketHandler - Delete bucket
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteBucket")
+	ctx := newContext(r, w, "DeleteBucket")
+
+	defer logger.AuditLog(w, r, "DeleteBucket", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -649,15 +757,21 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 	// Attempt to delete bucket.
 	if err := deleteBucket(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	globalNotificationSys.RemoveNotification(bucket)
 	globalPolicySys.Remove(bucket)
-	for nerr := range globalNotificationSys.DeleteBucket(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
+	globalNotificationSys.DeleteBucket(ctx, bucket)
+
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			// Deleting DNS entry failed, attempt to create the bucket again.
+			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 
 	// Write success response.

cmd/bucket-handlers_test.go

@@ -84,7 +84,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		// initialize httptest Recorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		// construct HTTP request for Get bucket location.
-		req, err := newTestSignedRequestV4("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, err := newTestSignedRequestV4("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -116,7 +116,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -220,7 +220,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		// construct HTTP request for HEAD bucket.
-		req, err := newTestSignedRequestV4("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, err := newTestSignedRequestV4("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -235,7 +235,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -437,7 +437,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 		// construct HTTP request for List multipart uploads endpoint.
 		u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads)
-		req, gerr := newTestSignedRequestV4("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+		req, gerr := newTestSignedRequestV4("GET", u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if gerr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", i+1, instanceType, gerr)
 		}
@@ -454,7 +454,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 		// construct HTTP request for PUT bucket policy endpoint.
 
 		// verify response for V2 signed HTTP request.
-		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -471,7 +471,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 	// construct HTTP request for List multipart uploads endpoint.
 	u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "")
-	req, err := newTestSignedRequestV4("GET", u, 0, nil, "", "") // Generate an anonymous request.
+	req, err := newTestSignedRequestV4("GET", u, 0, nil, "", "", nil) // Generate an anonymous request.
 	if err != nil {
 		t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", instanceType, err)
 	}
@@ -551,7 +551,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	for i, testCase := range testCases {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
-		req, lerr := newTestSignedRequestV4("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, lerr := newTestSignedRequestV4("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if lerr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListBucketsHandler: <ERROR> %v", i+1, instanceType, lerr)
 		}
@@ -568,7 +568,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 		// construct HTTP request for PUT bucket policy endpoint.
 
 		// verify response for V2 signed HTTP request.
-		reqV2, err := newTestSignedRequestV2("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -625,7 +625,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil)
+		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)
@@ -745,7 +745,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		// Generate a signed or anonymous request based on the testCase
 		if testCase.accessKey != "" {
 			req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
-				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
+				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey, nil)
 		} else {
 			req, err = newTestRequest("POST", getDeleteMultipleObjectsURL("", bucketName),
 				int64(len(testCase.objects)), bytes.NewReader(testCase.objects))
@@ -785,7 +785,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	nilBucket := "dummy-bucket"
 	nilObject := ""
 
-	nilReq, err := newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "")
+	nilReq, err := newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "", nil)
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}

cmd/bucket-notification-handlers.go

@@ -42,30 +42,32 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 // as per http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
 // It returns empty configuration if its not set.
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketNotification")
+	ctx := newContext(r, w, "GetBucketNotification")
+
+	defer logger.AuditLog(w, r, "GetBucketNotification", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	_, err := objAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -74,17 +76,21 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 	if err != nil {
 		// Ignore errNoSuchNotifications to comply with AWS S3.
 		if err != errNoSuchNotifications {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		nConfig = &event.Config{}
 	}
 
+	// If xml namespace is empty, set a default value before returning.
+	if nConfig.XMLNS == "" {
+		nConfig.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
+	}
+
 	notificationBytes, err := xml.Marshal(nConfig)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -94,16 +100,18 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 // PutBucketNotificationHandler - This HTTP handler stores given notification configuration as per
 // http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucketNotification")
+	ctx := newContext(r, w, "PutBucketNotification")
+
+	defer logger.AuditLog(w, r, "PutBucketNotification", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objectAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -111,19 +119,19 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	bucketName := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// PutBucketNotification always needs a Content-Length.
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -132,24 +140,21 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	if err != nil {
 		apiErr := ErrMalformedXML
 		if event.IsEventError(err) {
-			apiErr = toAPIErrorCode(err)
+			apiErr = toAPIErrorCode(ctx, err)
 		}
 
-		writeErrorResponse(w, apiErr, r.URL)
+		writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	if err = saveNotificationConfig(objectAPI, bucketName, config); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	rulesMap := config.ToRulesMap()
 	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
-	for nerr := range globalNotificationSys.PutBucketNotification(bucketName, rulesMap) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.PutBucketNotification(ctx, bucketName, rulesMap)
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -157,16 +162,18 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 // ListenBucketNotificationHandler - This HTTP handler sends events to the connected HTTP client.
 // Client should send prefix/suffix object name to match and events to watch as query parameters.
 func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListenBucketNotification")
+	ctx := newContext(r, w, "ListenBucketNotification")
+
+	defer logger.AuditLog(w, r, "ListenBucketNotification", mustGetClaimsFromToken(r))
 
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -174,7 +181,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	bucketName := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -182,11 +189,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	var prefix string
 	if len(values["prefix"]) > 1 {
-		writeErrorResponse(w, ErrFilterNamePrefix, r.URL)
+		writeErrorResponse(w, ErrFilterNamePrefix, r.URL, guessIsBrowserReq(r))
 	}
 	if len(values["prefix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -195,11 +202,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	var suffix string
 	if len(values["suffix"]) > 1 {
-		writeErrorResponse(w, ErrFilterNameSuffix, r.URL)
+		writeErrorResponse(w, ErrFilterNameSuffix, r.URL, guessIsBrowserReq(r))
 	}
 	if len(values["suffix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -212,7 +219,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	for _, s := range values["events"] {
 		eventName, err := event.ParseName(s)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -220,49 +227,51 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	}
 
 	if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	host, e := xnet.ParseHost(r.RemoteAddr)
-	logger.CriticalIf(ctx, e)
+	host, err := xnet.ParseHost(r.RemoteAddr)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
-	target, e := target.NewHTTPClientTarget(*host, w)
-	logger.CriticalIf(ctx, e)
+	target, err := target.NewHTTPClientTarget(*host, w)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
 
-	if err := globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
+	if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
 	defer globalNotificationSys.RemoveRulesMap(bucketName, rulesMap)
 
-	thisAddr, e := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-	logger.CriticalIf(ctx, e)
-
-	if err := SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
-		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	errCh := globalNotificationSys.ListenBucketNotification(bucketName, eventNames, pattern, target.ID(), *thisAddr)
-	for nerr := range errCh {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
+	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
+		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
 	}
 
+	globalNotificationSys.ListenBucketNotification(ctx, bucketName, eventNames, pattern, target.ID(), *thisAddr)
+
 	<-target.DoneCh
 
-	if err := RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
+	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 }

cmd/bucket-policy-handlers.go

@@ -38,11 +38,13 @@ const (
 // PutBucketPolicyHandler - This HTTP handler stores given bucket policy configuration as per
 // https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucketPolicy")
+	ctx := newContext(r, w, "PutBucketPolicy")
+
+	defer logger.AuditLog(w, r, "PutBucketPolicy", mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -50,51 +52,48 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Error out if Content-Length is missing.
 	// PutBucketPolicy always needs Content-Length.
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Error out if Content-Length is beyond allowed size.
 	if r.ContentLength > maxBucketPolicySize {
-		writeErrorResponse(w, ErrEntityTooLarge, r.URL)
+		writeErrorResponse(w, ErrEntityTooLarge, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Version in policy must not be empty
 	if bucketPolicy.Version == "" {
-		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	globalPolicySys.Set(bucket, *bucketPolicy)
-	for nerr := range globalNotificationSys.SetBucketPolicy(bucket, bucketPolicy) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.SetBucketPolicy(ctx, bucket, bucketPolicy)
 
 	// Success.
 	writeSuccessNoContent(w)
@@ -102,11 +101,13 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 // DeleteBucketPolicyHandler - This HTTP handler removes bucket policy configuration.
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteBucketPolicy")
+	ctx := newContext(r, w, "DeleteBucketPolicy")
+
+	defer logger.AuditLog(w, r, "DeleteBucketPolicy", mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -114,26 +115,23 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	globalPolicySys.Remove(bucket)
-	for nerr := range globalNotificationSys.RemoveBucketPolicy(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.RemoveBucketPolicy(ctx, bucket)
 
 	// Success.
 	writeSuccessNoContent(w)
@@ -141,11 +139,13 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 
 // GetBucketPolicyHandler - This HTTP handler returns bucket policy configuration.
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketPolicy")
+	ctx := newContext(r, w, "GetBucketPolicy")
+
+	defer logger.AuditLog(w, r, "GetBucketPolicy", mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -153,27 +153,26 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Read bucket access policy.
 	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	policyData, err := json.Marshal(bucketPolicy)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/bucket-policy-handlers_test.go

@@ -254,7 +254,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testCase.bucketName),
-			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey)
+			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -268,7 +268,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testCase.bucketName),
-			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey)
+			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -304,7 +304,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -346,7 +346,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -360,7 +360,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -417,7 +417,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("GET", getGetPolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -437,11 +437,13 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 		if recV4.Code != testCase.expectedRespStatus {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			var expectedPolicy *policy.Policy
+			expectedPolicy, err = policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}
-			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			var gotPolicy *policy.Policy
+			gotPolicy, err = policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}
@@ -454,7 +456,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("GET", getGetPolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -509,7 +511,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("GET", getGetPolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -589,7 +591,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -639,7 +641,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for Delete bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -661,7 +663,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -678,7 +680,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for Delete bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("DELETE", getDeletePolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -712,7 +714,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)

cmd/build-constants.go

@@ -22,6 +22,9 @@ var (
 	// GOPATH - GOPATH value at the time of build.
 	GOPATH = ""
 
+	// GOROOT - GOROOT value at the time of build.
+	GOROOT = ""
+
 	// Go get development tag.
 	goGetTag = "DEVELOPMENT.GOGET"
 

cmd/certs.go

@@ -71,9 +71,13 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	// Get all CA file names.
 	var caFiles []string
 	fis, err := readDir(certsCAsDir)
-	if err != nil {
+	if err != nil && err != errFileNotFound {
 		return nil, err
 	}
+	// Return success if CA's directory is missing.
+	if err == errFileNotFound {
+		return nil, nil
+	}
 	for _, fi := range fis {
 		// Skip all directories.
 		if hasSuffix(fi, slashSeparator) {
@@ -86,10 +90,11 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 		return nil, nil
 	}
 
-	rootCAs, err := x509.SystemCertPool()
-	if err != nil {
-		// In some systems like Windows, system cert pool is not supported.
-		// Hence we create a new cert pool.
+	rootCAs, _ := x509.SystemCertPool()
+	if rootCAs == nil {
+		// In some systems (like Windows) system cert pool is
+		// not supported or no certificates are present on the
+		// system - so we create a new cert pool.
 		rootCAs = x509.NewCertPool()
 	}
 
@@ -149,24 +154,20 @@ func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	return cert, nil
 }
 
-func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, c *certs.Certs, secureConn bool, err error) {
+func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
-		return nil, nil, nil, false, nil
+		return nil, nil, false, nil
 	}
 
 	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
-		return nil, nil, nil, false, err
+		return nil, nil, false, err
 	}
 
 	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
 	if err != nil {
-		return nil, nil, nil, false, err
-	}
-
-	if rootCAs, err = getRootCAs(getCADir()); err != nil {
-		return nil, nil, nil, false, err
+		return nil, nil, false, err
 	}
 
 	secureConn = true
-	return x509Certs, rootCAs, c, secureConn, nil
+	return x509Certs, c, secureConn, nil
 }

cmd/certs_test.go

@@ -223,7 +223,8 @@ func TestGetRootCAs(t *testing.T) {
 		certCAsDir  string
 		expectedErr error
 	}{
-		{"nonexistent-dir", errFileNotFound},
+		// ignores non-existent directories.
+		{"nonexistent-dir", nil},
 		// Ignores directories.
 		{dir1, nil},
 		// Ignore empty directory.

cmd/common-main.go

@@ -17,16 +17,26 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"errors"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
+	etcd "github.com/coreos/etcd/clientv3"
+	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/target/console"
+	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Check for updates and print a notification message
@@ -41,24 +51,42 @@ func checkUpdate(mode string) {
 	}
 }
 
-func initConfig() {
-	// Config file does not exist, we create it fresh and return upon success.
-	if isFile(getConfigFile()) {
-		logger.FatalIf(migrateConfig(), "Config migration failed")
-		logger.FatalIf(loadConfig(), "Unable to load the configuration file")
-	} else {
-		logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time")
-		logger.Info("Created minio configuration file successfully at " + getConfigDir())
+// Load logger targets based on user's configuration
+func loadLoggers() {
+	auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
+	if ok {
+		// Enable audit HTTP logging through ENV.
+		logger.AddAuditTarget(http.New(auditEndpoint, NewCustomHTTPTransport()))
 	}
+
+	loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
+	if ok {
+		// Enable HTTP logging through ENV.
+		logger.AddTarget(http.New(loggerEndpoint, NewCustomHTTPTransport()))
+	} else {
+		for _, l := range globalServerConfig.Logger.HTTP {
+			if l.Enabled {
+				// Enable http logging
+				logger.AddTarget(http.New(l.Endpoint, NewCustomHTTPTransport()))
+			}
+		}
+	}
+
+	if globalServerConfig.Logger.Console.Enabled {
+		// Enable console logging
+		logger.AddTarget(console.New())
+	}
+
 }
 
 func handleCommonCmdArgs(ctx *cli.Context) {
 
 	var configDir string
 
-	if ctx.IsSet("config-dir") {
+	switch {
+	case ctx.IsSet("config-dir"):
 		configDir = ctx.String("config-dir")
-	} else if ctx.GlobalIsSet("config-dir") {
+	case ctx.GlobalIsSet("config-dir"):
 		configDir = ctx.GlobalString("config-dir")
 		// cli package does not expose parent's "config-dir" option.  Below code is workaround.
 		if configDir == "" || configDir == getConfigDir() {
@@ -66,7 +94,7 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 				configDir = ctx.Parent().GlobalString("config-dir")
 			}
 		}
-	} else {
+	default:
 		// Neither local nor global config-dir option is provided.  In this case, try to use
 		// default config directory.
 		configDir = getConfigDir()
@@ -85,10 +113,23 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	setConfigDir(configDirAbs)
 }
 
+// Parses the given compression exclude list `extensions` or `content-types`.
+func parseCompressIncludes(includes []string) ([]string, error) {
+	for _, e := range includes {
+		if len(e) == 0 {
+			return nil, uiErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
+		}
+	}
+	return includes, nil
+}
+
 func handleCommonEnvVars() {
+	compressEnvDelimiter := ","
 	// Start profiler if env is set.
 	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
-		globalProfiler = startProfiler(profiler)
+		var err error
+		globalProfiler, err = startProfiler(profiler, "")
+		logger.FatalIf(err, "Unable to setup a profiler")
 	}
 
 	accessKey := os.Getenv("MINIO_ACCESS_KEY")
@@ -98,6 +139,7 @@ func handleCommonEnvVars() {
 		if err != nil {
 			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
 		}
+		cred.Expiration = timeSentinel
 
 		// credential Envs are set globally.
 		globalIsEnvCreds = true
@@ -107,7 +149,7 @@ func handleCommonEnvVars() {
 	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
 		browserFlag, err := ParseBoolFlag(browser)
 		if err != nil {
-			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Unable to validate MINIO_BROWSER environment variable")
+			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -123,7 +165,75 @@ func handleCommonEnvVars() {
 		logger.FatalIf(err, "error opening file %s", traceFile)
 	}
 
+	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	if ok {
+		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
+
+		var etcdSecure bool
+		for _, endpoint := range etcdEndpoints {
+			u, err := xnet.ParseURL(endpoint)
+			if err != nil {
+				logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+			}
+			// If one of the endpoint is https, we will use https directly.
+			etcdSecure = etcdSecure || u.Scheme == "https"
+		}
+
+		var err error
+		if etcdSecure {
+			// This is only to support client side certificate authentication
+			// https://coreos.com/etcd/docs/latest/op-guide/security.html
+			etcdClientCertFile, ok1 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT")
+			etcdClientCertKey, ok2 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT_KEY")
+			var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+			if ok1 && ok2 {
+				getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+					cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
+					return &cert, terr
+				}
+			}
+			globalEtcdClient, err = etcd.New(etcd.Config{
+				Endpoints:         etcdEndpoints,
+				DialTimeout:       defaultDialTimeout,
+				DialKeepAliveTime: defaultDialKeepAlive,
+				TLS: &tls.Config{
+					RootCAs:              globalRootCAs,
+					GetClientCertificate: getClientCertificate,
+				},
+			})
+		} else {
+			globalEtcdClient, err = etcd.New(etcd.Config{
+				Endpoints:         etcdEndpoints,
+				DialTimeout:       defaultDialTimeout,
+				DialKeepAliveTime: defaultDialKeepAlive,
+			})
+		}
+		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+	}
+
 	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
+	if globalDomainName != "" {
+		if _, ok = dns2.IsDomainName(globalDomainName); !ok {
+			logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", globalDomainName), "Invalid MINIO_DOMAIN value in environment variable")
+		}
+	}
+
+	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if ok {
+		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
+		globalDomainIPs = set.NewStringSet()
+		for i, ip := range minioEndpoints {
+			if net.ParseIP(ip) == nil {
+				logger.FatalIf(errInvalidArgument, "Unable to initialize Minio server with invalid MINIO_PUBLIC_IPS[%d]: %s", i, ip)
+			}
+			globalDomainIPs.Add(ip)
+		}
+	}
+	if globalDomainName != "" && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+		var err error
+		globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIPs, globalMinioPort, globalEtcdClient)
+		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainName)
+	}
 
 	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
 		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
@@ -150,6 +260,16 @@ func handleCommonEnvVars() {
 		globalCacheExpiry = expiry
 	}
 
+	if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" {
+		maxUse, err := strconv.Atoi(maxUseStr)
+		if err != nil {
+			logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr)
+		}
+		// maxUse should be a valid percentage.
+		if maxUse > 0 && maxUse <= 100 {
+			globalCacheMaxUse = maxUse
+		}
+	}
 	// In place update is true by default if the MINIO_UPDATE is not set
 	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
 	// in-place update is off.
@@ -189,7 +309,7 @@ func handleCommonEnvVars() {
 	if worm := os.Getenv("MINIO_WORM"); worm != "" {
 		wormFlag, err := ParseBoolFlag(worm)
 		if err != nil {
-			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Unable to validate MINIO_WORM environment variable")
+			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
 		}
 
 		// worm Envs are set globally, this does not represent
@@ -197,4 +317,42 @@ func handleCommonEnvVars() {
 		globalIsEnvWORM = true
 		globalWORMEnabled = bool(wormFlag)
 	}
+
+	kmsConf, err := crypto.NewVaultConfig()
+	if err != nil {
+		logger.Fatal(err, "Unable to initialize hashicorp vault")
+	}
+	if kmsConf.Vault.Endpoint != "" {
+		kms, err := crypto.NewVault(kmsConf)
+		if err != nil {
+			logger.Fatal(err, "Unable to initialize KMS")
+		}
+		globalKMS = kms
+		globalKMSKeyID = kmsConf.Vault.Key.Name
+		globalKMSConfig = kmsConf
+	}
+
+	if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
+		globalIsCompressionEnabled = strings.EqualFold(compress, "true")
+	}
+
+	compressExtensions := os.Getenv("MINIO_COMPRESS_EXTENSIONS")
+	compressMimeTypes := os.Getenv("MINIO_COMPRESS_MIMETYPES")
+	if compressExtensions != "" || compressMimeTypes != "" {
+		globalIsEnvCompression = true
+		if compressExtensions != "" {
+			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter))
+			if err != nil {
+				logger.Fatal(err, "Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", extensions)
+			}
+			globalCompressExtensions = extensions
+		}
+		if compressMimeTypes != "" {
+			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter))
+			if err != nil {
+				logger.Fatal(err, "Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", contenttypes)
+			}
+			globalCompressMimeTypes = contenttypes
+		}
+	}
 }

cmd/config-common.go

@@ -0,0 +1,133 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+)
+
+var errConfigNotFound = errors.New("config file not found")
+
+func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
+	var buffer bytes.Buffer
+	// Read entire content by setting size to -1
+	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
+		// Treat object not found as config not found.
+		if isErrObjectNotFound(err) {
+			return nil, errConfigNotFound
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
+		logger.LogIf(ctx, err)
+		return nil, err
+	}
+
+	// Return config not found on empty content.
+	if buffer.Len() == 0 {
+		return nil, errConfigNotFound
+	}
+
+	return buffer.Bytes(), nil
+}
+
+func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	_, err := client.Delete(ctx, configFile)
+	return err
+}
+
+func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+}
+
+func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
+	_, err := client.Put(ctx, configFile, string(data))
+	return err
+}
+
+func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
+	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
+	if err != nil {
+		return err
+	}
+
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), nil, ObjectOptions{})
+	return err
+}
+
+func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) ([]byte, error) {
+	resp, err := client.Get(ctx, configFile)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Count == 0 {
+		return nil, errConfigNotFound
+	}
+	for _, ev := range resp.Kvs {
+		if string(ev.Key) == configFile {
+			return ev.Value, nil
+		}
+	}
+	return nil, errConfigNotFound
+}
+
+// watchConfig - watches for changes on `configFile` on etcd and loads them.
+func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
+	if globalEtcdClient != nil {
+		for watchResp := range globalEtcdClient.Watch(context.Background(), configFile) {
+			for _, event := range watchResp.Events {
+				if event.IsModify() || event.IsCreate() {
+					loadCfgFn(objAPI)
+				}
+			}
+		}
+	}
+}
+
+func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	resp, err := globalEtcdClient.Get(ctx, configFile)
+	if err != nil {
+		return err
+	}
+	if resp.Count == 0 {
+		return errConfigNotFound
+	}
+	return nil
+}
+
+func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	if globalEtcdClient != nil {
+		return checkConfigEtcd(ctx, globalEtcdClient, configFile)
+	}
+
+	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
+		// Treat object not found as config not found.
+		if isErrObjectNotFound(err) {
+			return errConfigNotFound
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
+		logger.LogIf(ctx, err)
+		return err
+	}
+	return nil
+}

cmd/config-current.go

@@ -17,17 +17,21 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"os"
 	"reflect"
 	"sync"
 
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
-
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
-	"github.com/minio/minio/pkg/quick"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Steps to move from version N to version N+1
@@ -39,9 +43,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "25"
+const serverConfigVersion = "32"
 
-type serverConfig = serverConfigV25
+type serverConfig = serverConfigV32
 
 var (
 	// globalServerConfig server config.
@@ -62,11 +66,21 @@ func (s *serverConfig) SetRegion(region string) {
 
 // GetRegion get current region.
 func (s *serverConfig) GetRegion() string {
+	if globalIsEnvRegion {
+		return globalServerRegion
+	}
+	if s == nil {
+		return ""
+	}
 	return s.Region
 }
 
 // SetCredential sets new credential and returns the previous credential.
 func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
+	if creds.IsValid() && globalActiveCred.IsValid() {
+		globalActiveCred = creds
+	}
+
 	// Save previous credential.
 	prevCred = s.Credential
 
@@ -79,15 +93,12 @@ func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Cred
 
 // GetCredentials get current credentials.
 func (s *serverConfig) GetCredential() auth.Credentials {
+	if globalActiveCred.IsValid() {
+		return globalActiveCred
+	}
 	return s.Credential
 }
 
-// SetBrowser set if browser is enabled.
-func (s *serverConfig) SetBrowser(b bool) {
-	// Set the new value.
-	s.Browser = BoolFlag(b)
-}
-
 // SetWorm set if worm is enabled.
 func (s *serverConfig) SetWorm(b bool) {
 	// Set the new value.
@@ -102,35 +113,292 @@ func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
 // GetStorageClass reads storage class fields from current config.
 // It returns the standard and reduced redundancy storage class struct
 func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
+	if globalIsStorageClass {
+		return globalStandardStorageClass, globalRRStorageClass
+	}
+	if s == nil {
+		return storageClass{}, storageClass{}
+	}
 	return s.StorageClass.Standard, s.StorageClass.RRS
 }
 
-// GetBrowser get current credentials.
-func (s *serverConfig) GetBrowser() bool {
-	return bool(s.Browser)
-}
-
 // GetWorm get current credentials.
 func (s *serverConfig) GetWorm() bool {
+	if globalIsEnvWORM {
+		return globalWORMEnabled
+	}
+	if s == nil {
+		return false
+	}
 	return bool(s.Worm)
 }
 
 // SetCacheConfig sets the current cache config
-func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int) {
+func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxuse int) {
 	s.Cache.Drives = drives
 	s.Cache.Exclude = exclude
 	s.Cache.Expiry = expiry
+	s.Cache.MaxUse = maxuse
 }
 
 // GetCacheConfig gets the current cache config
 func (s *serverConfig) GetCacheConfig() CacheConfig {
+	if globalIsDiskCacheEnabled {
+		return CacheConfig{
+			Drives:  globalCacheDrives,
+			Exclude: globalCacheExcludes,
+			Expiry:  globalCacheExpiry,
+			MaxUse:  globalCacheMaxUse,
+		}
+	}
+	if s == nil {
+		return CacheConfig{}
+	}
 	return s.Cache
 }
 
-// Save config.
-func (s *serverConfig) Save() error {
-	// Save config file.
-	return quick.Save(getConfigFile(), s)
+func (s *serverConfig) Validate() error {
+	if s == nil {
+		return nil
+	}
+	if s.Version != serverConfigVersion {
+		return fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, s.Version)
+	}
+
+	// Validate credential fields only when
+	// they are not set via the environment
+	// Error out if global is env credential is not set and config has invalid credential
+	if !globalIsEnvCreds && !s.Credential.IsValid() {
+		return errors.New("invalid credential in config file")
+	}
+
+	// Region: nothing to validate
+	// Worm, Cache and StorageClass values are already validated during json unmarshal
+	for _, v := range s.Notify.AMQP {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("amqp: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.Elasticsearch {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("elasticsearch: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.Kafka {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("kafka: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.MQTT {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("mqtt: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.MySQL {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("mysql: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.NATS {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("nats: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.NSQ {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("nsq: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.PostgreSQL {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("postgreSQL: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.Redis {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("redis: %s", err)
+		}
+	}
+
+	for _, v := range s.Notify.Webhook {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("webhook: %s", err)
+		}
+	}
+
+	return nil
+}
+
+// SetCompressionConfig sets the current compression config
+func (s *serverConfig) SetCompressionConfig(extensions []string, mimeTypes []string) {
+	s.Compression.Extensions = extensions
+	s.Compression.MimeTypes = mimeTypes
+	s.Compression.Enabled = globalIsCompressionEnabled
+}
+
+// GetCompressionConfig gets the current compression config
+func (s *serverConfig) GetCompressionConfig() compressionConfig {
+	return s.Compression
+}
+
+func (s *serverConfig) loadFromEnvs() {
+	// If env is set override the credentials from config file.
+	if globalIsEnvCreds {
+		s.SetCredential(globalActiveCred)
+	}
+
+	if globalIsEnvWORM {
+		s.SetWorm(globalWORMEnabled)
+	}
+
+	if globalIsEnvRegion {
+		s.SetRegion(globalServerRegion)
+	}
+
+	if globalIsStorageClass {
+		s.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
+	}
+
+	if globalIsDiskCacheEnabled {
+		s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
+	}
+
+	if globalKMS != nil {
+		s.KMS = globalKMSConfig
+	}
+
+	if globalIsEnvCompression {
+		s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
+	}
+
+	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
+		if u, err := xnet.ParseURL(jwksURL); err == nil {
+			s.OpenID.JWKS.URL = u
+			s.OpenID.JWKS.PopulatePublicKey()
+		}
+	}
+
+	if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
+		if u, err := xnet.ParseURL(opaURL); err == nil {
+			s.Policy.OPA.URL = u
+			s.Policy.OPA.AuthToken = os.Getenv("MINIO_IAM_OPA_AUTHTOKEN")
+		}
+	}
+}
+
+// TestNotificationTargets tries to establish connections to all notification
+// targets when enabled. This is a good way to make sure all configurations
+// set by the user can work.
+func (s *serverConfig) TestNotificationTargets() error {
+	for k, v := range s.Notify.AMQP {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewAMQPTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("amqp(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Elasticsearch {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewElasticsearchTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("elasticsearch(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Kafka {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewKafkaTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("kafka(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.MQTT {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewMQTTTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("mqtt(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.MySQL {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewMySQLTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("mysql(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.NATS {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewNATSTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("nats(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.NSQ {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewNSQTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("nsq(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.PostgreSQL {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewPostgreSQLTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("postgreSQL(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Redis {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewRedisTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("redis(%s): %s", k, err.Error())
+		}
+		t.Close()
+
+	}
+
+	return nil
 }
 
 // Returns the string describing a difference with the given
@@ -144,18 +412,18 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "Credential configuration differs"
 	case s.Region != t.Region:
 		return "Region configuration differs"
-	case s.Browser != t.Browser:
-		return "Browser configuration differs"
-	case s.Domain != t.Domain:
-		return "Domain configuration differs"
 	case s.StorageClass != t.StorageClass:
 		return "StorageClass configuration differs"
 	case !reflect.DeepEqual(s.Cache, t.Cache):
 		return "Cache configuration differs"
+	case !reflect.DeepEqual(s.Compression, t.Compression):
+		return "Compression configuration differs"
 	case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
 		return "AMQP Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
 		return "NATS Notification configuration differs"
+	case !reflect.DeepEqual(s.Notify.NSQ, t.Notify.NSQ):
+		return "NSQ Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.Elasticsearch, t.Notify.Elasticsearch):
 		return "ElasticSearch Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.Redis, t.Notify.Redis):
@@ -170,6 +438,10 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "MySQL Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.MQTT, t.Notify.MQTT):
 		return "MQTT Notification configuration differs"
+	case !reflect.DeepEqual(s.Logger, t.Logger):
+		return "Logger configuration differs"
+	case !reflect.DeepEqual(s.KMS, t.KMS):
+		return "KMS configuration differs"
 	case reflect.DeepEqual(s, t):
 		return ""
 	default:
@@ -187,7 +459,6 @@ func newServerConfig() *serverConfig {
 		Version:    serverConfigVersion,
 		Credential: cred,
 		Region:     globalMinioDefaultRegion,
-		Browser:    true,
 		StorageClass: storageClassConfig{
 			Standard: storageClass{},
 			RRS:      storageClass{},
@@ -196,8 +467,15 @@ func newServerConfig() *serverConfig {
 			Drives:  []string{},
 			Exclude: []string{},
 			Expiry:  globalCacheExpiry,
+			MaxUse:  globalCacheMaxUse,
 		},
+		KMS:    crypto.KMSConfig{},
 		Notify: notifier{},
+		Compression: compressionConfig{
+			Enabled:    false,
+			Extensions: globalCompressExtensions,
+			MimeTypes:  globalCompressMimeTypes,
+		},
 	}
 
 	// Make sure to initialize notification configs.
@@ -211,6 +489,8 @@ func newServerConfig() *serverConfig {
 	srvCfg.Notify.Redis["1"] = target.RedisArgs{}
 	srvCfg.Notify.NATS = make(map[string]target.NATSArgs)
 	srvCfg.Notify.NATS["1"] = target.NATSArgs{}
+	srvCfg.Notify.NSQ = make(map[string]target.NSQArgs)
+	srvCfg.Notify.NSQ["1"] = target.NSQArgs{}
 	srvCfg.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
 	srvCfg.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{}
 	srvCfg.Notify.MySQL = make(map[string]target.MySQLArgs)
@@ -223,161 +503,155 @@ func newServerConfig() *serverConfig {
 	srvCfg.Cache.Drives = make([]string, 0)
 	srvCfg.Cache.Exclude = make([]string, 0)
 	srvCfg.Cache.Expiry = globalCacheExpiry
+	srvCfg.Cache.MaxUse = globalCacheMaxUse
+
+	// Console logging is on by default
+	srvCfg.Logger.Console.Enabled = true
+	// Create an example of HTTP logger
+	srvCfg.Logger.HTTP = make(map[string]loggerHTTP)
+	srvCfg.Logger.HTTP["target1"] = loggerHTTP{Endpoint: "https://username:password@example.com/api"}
+
 	return srvCfg
 }
 
-// newConfig - initialize a new server config, saves env parameters if
+func (s *serverConfig) loadToCachedConfigs() {
+	if !globalIsEnvCreds {
+		globalActiveCred = s.GetCredential()
+	}
+	if !globalIsEnvWORM {
+		globalWORMEnabled = s.GetWorm()
+	}
+	if !globalIsEnvRegion {
+		globalServerRegion = s.GetRegion()
+	}
+	if !globalIsStorageClass {
+		globalStandardStorageClass, globalRRStorageClass = s.GetStorageClass()
+	}
+	if !globalIsDiskCacheEnabled {
+		cacheConf := s.GetCacheConfig()
+		globalCacheDrives = cacheConf.Drives
+		globalCacheExcludes = cacheConf.Exclude
+		globalCacheExpiry = cacheConf.Expiry
+		globalCacheMaxUse = cacheConf.MaxUse
+	}
+	if globalKMS == nil {
+		globalKMSConfig = s.KMS
+		if kms, err := crypto.NewVault(globalKMSConfig); err == nil {
+			globalKMS = kms
+			globalKMSKeyID = globalKMSConfig.Vault.Key.Name
+		}
+	}
+
+	if !globalIsCompressionEnabled {
+		compressionConf := s.GetCompressionConfig()
+		globalCompressExtensions = compressionConf.Extensions
+		globalCompressMimeTypes = compressionConf.MimeTypes
+		globalIsCompressionEnabled = compressionConf.Enabled
+	}
+
+	if globalIAMValidators == nil {
+		globalIAMValidators = getAuthValidators(s)
+	}
+
+	if globalPolicyOPA == nil {
+		if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
+			globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
+				URL:         s.Policy.OPA.URL,
+				AuthToken:   s.Policy.OPA.AuthToken,
+				Transport:   NewCustomHTTPTransport(),
+				CloseRespFn: CloseResponse,
+			})
+		}
+	}
+}
+
+// newSrvConfig - initialize a new server config, saves env parameters if
 // found, otherwise use default parameters
-func newConfig() error {
+func newSrvConfig(objAPI ObjectLayer) error {
 	// Initialize server config.
 	srvCfg := newServerConfig()
 
-	// If env is set override the credentials from config file.
-	if globalIsEnvCreds {
-		srvCfg.SetCredential(globalActiveCred)
-	}
+	// Override any values from ENVs.
+	srvCfg.loadFromEnvs()
 
-	if globalIsEnvBrowser {
-		srvCfg.SetBrowser(globalIsBrowserEnabled)
-	}
-
-	if globalIsEnvWORM {
-		srvCfg.SetWorm(globalWORMEnabled)
-	}
-
-	if globalIsEnvRegion {
-		srvCfg.SetRegion(globalServerRegion)
-	}
-
-	if globalIsEnvDomainName {
-		srvCfg.Domain = globalDomainName
-	}
-
-	if globalIsStorageClass {
-		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
-	}
-
-	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
-	}
+	// Load values to cached global values.
+	srvCfg.loadToCachedConfigs()
 
 	// hold the mutex lock before a new config is assigned.
-	// Save the new config globally.
-	// unlock the mutex.
 	globalServerConfigMu.Lock()
 	globalServerConfig = srvCfg
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return globalServerConfig.Save()
+	return saveServerConfig(context.Background(), objAPI, globalServerConfig)
 }
 
 // getValidConfig - returns valid server configuration
-func getValidConfig() (*serverConfig, error) {
-	srvCfg := &serverConfig{
-		Region:  globalMinioDefaultRegion,
-		Browser: true,
-	}
-
-	if _, err := quick.Load(getConfigFile(), srvCfg); err != nil {
+func getValidConfig(objAPI ObjectLayer) (*serverConfig, error) {
+	srvCfg, err := readServerConfig(context.Background(), objAPI)
+	if err != nil {
 		return nil, err
 	}
 
-	if srvCfg.Version != serverConfigVersion {
-		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
-	}
-
-	// Validate credential fields only when
-	// they are not set via the environment
-	// Error out if global is env credential is not set and config has invalid credential
-	if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
-		return nil, errors.New("invalid credential in config file " + getConfigFile())
-	}
-
-	return srvCfg, nil
+	return srvCfg, srvCfg.Validate()
 }
 
 // loadConfig - loads a new config from disk, overrides params from env
 // if found and valid
-func loadConfig() error {
-	srvCfg, err := getValidConfig()
+func loadConfig(objAPI ObjectLayer) error {
+	srvCfg, err := getValidConfig(objAPI)
 	if err != nil {
 		return uiErrInvalidConfig(nil).Msg(err.Error())
 	}
 
-	// If env is set override the credentials from config file.
-	if globalIsEnvCreds {
-		srvCfg.SetCredential(globalActiveCred)
-	}
+	// Override any values from ENVs.
+	srvCfg.loadFromEnvs()
 
-	if globalIsEnvBrowser {
-		srvCfg.SetBrowser(globalIsBrowserEnabled)
-	}
-
-	if globalIsEnvRegion {
-		srvCfg.SetRegion(globalServerRegion)
-	}
-
-	if globalIsEnvDomainName {
-		srvCfg.Domain = globalDomainName
-	}
-
-	if globalIsStorageClass {
-		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
-	}
-
-	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
-	}
+	// Load values to cached global values.
+	srvCfg.loadToCachedConfigs()
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()
 	globalServerConfig = srvCfg
-	if !globalIsEnvCreds {
-		globalActiveCred = globalServerConfig.GetCredential()
-	}
-	if !globalIsEnvBrowser {
-		globalIsBrowserEnabled = globalServerConfig.GetBrowser()
-	}
-	if !globalIsEnvWORM {
-		globalWORMEnabled = globalServerConfig.GetWorm()
-	}
-	if !globalIsEnvRegion {
-		globalServerRegion = globalServerConfig.GetRegion()
-	}
-	if !globalIsEnvDomainName {
-		globalDomainName = globalServerConfig.Domain
-	}
-	if !globalIsStorageClass {
-		globalStandardStorageClass, globalRRStorageClass = globalServerConfig.GetStorageClass()
-	}
-	if !globalIsDiskCacheEnabled {
-		cacheConf := globalServerConfig.GetCacheConfig()
-		globalCacheDrives = cacheConf.Drives
-		globalCacheExcludes = cacheConf.Exclude
-		globalCacheExpiry = cacheConf.Expiry
-	}
 	globalServerConfigMu.Unlock()
 
 	return nil
 }
 
+// getAuthValidators - returns ValidatorList which contains
+// enabled providers in server config.
+// A new authentication provider is added like below
+// * Add a new provider in pkg/iam/validator package.
+func getAuthValidators(config *serverConfig) *validator.Validators {
+	validators := validator.NewValidators()
+
+	if config.OpenID.JWKS.URL != nil {
+		validators.Add(validator.NewJWT(config.OpenID.JWKS))
+	}
+
+	return validators
+}
+
 // getNotificationTargets - returns TargetList which contains enabled targets in serverConfig.
 // A new notification target is added like below
 // * Add a new target in pkg/event/target package.
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
-func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
+func getNotificationTargets(config *serverConfig) *event.TargetList {
 	targetList := event.NewTargetList()
-
+	if config == nil {
+		return targetList
+	}
 	for id, args := range config.Notify.AMQP {
 		if args.Enable {
 			newTarget, err := target.NewAMQPTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -386,10 +660,14 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewElasticsearchTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+
 			}
 		}
 	}
@@ -398,10 +676,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewKafkaTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -410,10 +690,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewMQTTTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -422,10 +704,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewMySQLTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -434,10 +718,26 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewNATSTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+			}
+		}
+	}
+
+	for id, args := range config.Notify.NSQ {
+		if args.Enable {
+			newTarget, err := target.NewNSQTarget(id, args)
+			if err != nil {
+				logger.LogIf(context.Background(), err)
+				continue
+			}
+			if err = targetList.Add(newTarget); err != nil {
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -446,10 +746,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewPostgreSQLTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -458,10 +760,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewRedisTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -470,10 +774,11 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget := target.NewWebhookTarget(id, args)
 			if err := targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
 
-	return targetList, nil
+	return targetList
 }

cmd/config-current_test.go

@@ -17,9 +17,9 @@
 package cmd
 
 import (
-	"io/ioutil"
+	"context"
 	"os"
-	"path/filepath"
+	"path"
 	"testing"
 
 	"github.com/minio/minio/pkg/auth"
@@ -27,12 +27,15 @@ import (
 )
 
 func TestServerConfig(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	objLayer, fsDir, err := prepareFS()
 	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
 		t.Fatalf("Init Test config failed")
 	}
-	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
 
 	if globalServerConfig.GetRegion() != globalMinioDefaultRegion {
 		t.Errorf("Expecting region `us-east-1` found %s", globalServerConfig.GetRegion())
@@ -49,16 +52,12 @@ func TestServerConfig(t *testing.T) {
 		t.Errorf("Expecting version %s found %s", globalServerConfig.GetVersion(), serverConfigVersion)
 	}
 
-	// Attempt to save.
-	if err := globalServerConfig.Save(); err != nil {
+	if err := saveServerConfig(context.Background(), objLayer, globalServerConfig); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 
-	// Do this only once here.
-	setConfigDir(rootPath)
-
 	// Initialize server config.
-	if err := loadConfig(); err != nil {
+	if err := loadConfig(objLayer); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 }
@@ -68,6 +67,9 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	os.Setenv("MINIO_BROWSER", "off")
 	defer os.Unsetenv("MINIO_BROWSER")
 
+	os.Setenv("MINIO_WORM", "on")
+	defer os.Unsetenv("MINIO_WORM")
+
 	os.Setenv("MINIO_ACCESS_KEY", "minio")
 	defer os.Unsetenv("MINIO_ACCESS_KEY")
 
@@ -82,61 +84,70 @@ func TestServerConfigWithEnvs(t *testing.T) {
 
 	defer resetGlobalIsEnvs()
 
-	// Get test root.
-	rootPath, err := getTestRoot()
+	objLayer, fsDir, err := prepareFS()
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
+	defer os.RemoveAll(fsDir)
+
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = objLayer
+	globalObjLayerMutex.Unlock()
 
 	serverHandleEnvVars()
 
-	// Do this only once here.
-	setConfigDir(rootPath)
-
 	// Init config
-	initConfig()
+	initConfig(objLayer)
 
-	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
-
-	// Check if serverConfig has
-	if globalServerConfig.GetBrowser() {
-		t.Errorf("Expecting browser is set to false found %v", globalServerConfig.GetBrowser())
+	// Check if serverConfig has browser disabled
+	if globalIsBrowserEnabled {
+		t.Error("Expected browser to be disabled but it is not")
 	}
 
-	// Check if serverConfig has
+	// Check if serverConfig returns WORM config from the env
+	if !globalServerConfig.GetWorm() {
+		t.Error("Expected WORM to be enabled but it is not")
+	}
+
+	// Check if serverConfig has region from the environment
 	if globalServerConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expecting region to be \"us-west-1\" found %v", globalServerConfig.GetRegion())
+		t.Errorf("Expected region to be \"us-west-1\", found %v", globalServerConfig.GetRegion())
 	}
 
-	// Check if serverConfig has
+	// Check if serverConfig has credentials from the environment
 	cred := globalServerConfig.GetCredential()
 
 	if cred.AccessKey != "minio" {
-		t.Errorf("Expecting access key to be `minio` found %s", cred.AccessKey)
+		t.Errorf("Expected access key to be `minio`, found %s", cred.AccessKey)
 	}
 
 	if cred.SecretKey != "minio123" {
-		t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
+		t.Errorf("Expected access key to be `minio123`, found %s", cred.SecretKey)
 	}
 
-	if globalServerConfig.Domain != "domain.com" {
-		t.Errorf("Expecting Domain to be `domain.com` found " + globalServerConfig.Domain)
+	// Check if serverConfig has the correct domain
+	if globalDomainName != "domain.com" {
+		t.Errorf("Expected Domain to be `domain.com`, found " + globalDomainName)
 	}
 }
 
 // Tests config validator..
 func TestValidateConfig(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	objLayer, fsDir, err := prepareFS()
 	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
 		t.Fatalf("Init Test config failed")
 	}
 
-	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
-
-	configPath := filepath.Join(rootPath, minioConfigFile)
-
+	configPath := path.Join(minioConfigPrefix, minioConfigFile)
 	v := serverConfigVersion
 
 	testCases := []struct {
@@ -174,66 +185,69 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
 
 		// Test 11 - Test AMQP
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
 
 		// Test 12 - Test NATS
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
 
 		// Test 13 - Test ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
 
 		// Test 14 - Test Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
 
 		// Test 15 - Test PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
 		// Test 16 - Test Kafka
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
 
 		// Test 17 - Test Webhook
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
 
 		// Test 18 - Test MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
 		// Test 19 - Test Format for MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
 		// Test 20 - Test valid Format for MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
 		// Test 21 - Test Format for PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
 		// Test 22 - Test valid Format for PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
 		// Test 23 - Test Format for ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
 
 		// Test 24 - Test valid Format for ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
 
 		// Test 25 - Test Format for Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
 
 		// Test 26 - Test valid Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
 
 		// Test 27 - Test MQTT
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
+
+		// Test 28 - Test NSQ
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": ""} }}}`, false},
 	}
 
 	for i, testCase := range testCases {
-		if werr := ioutil.WriteFile(configPath, []byte(testCase.configData), 0700); werr != nil {
-			t.Fatal(werr)
+		if err = saveConfig(context.Background(), objLayer, configPath, []byte(testCase.configData)); err != nil {
+			t.Fatal(err)
 		}
-		_, verr := getValidConfig()
-		if testCase.shouldPass && verr != nil {
-			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, verr)
+		_, err = getValidConfig(objLayer)
+		if testCase.shouldPass && err != nil {
+			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, err)
 		}
-		if !testCase.shouldPass && verr == nil {
+		if !testCase.shouldPass && err == nil {
 			t.Errorf("Test %d, should fail but it succeeded.", i+1)
 		}
 	}
@@ -249,76 +263,98 @@ func TestConfigDiff(t *testing.T) {
 		{&serverConfig{}, nil, "Given configuration is empty"},
 		// 2
 		{
-			&serverConfig{Credential: auth.Credentials{"u1", "p1"}},
-			&serverConfig{Credential: auth.Credentials{"u1", "p2"}},
+			&serverConfig{Credential: auth.Credentials{
+				AccessKey:  "u1",
+				SecretKey:  "p1",
+				Expiration: timeSentinel,
+			}},
+			&serverConfig{Credential: auth.Credentials{
+				AccessKey:  "u1",
+				SecretKey:  "p2",
+				Expiration: timeSentinel,
+			}},
 			"Credential configuration differs",
 		},
 		// 3
 		{&serverConfig{Region: "us-east-1"}, &serverConfig{Region: "us-west-1"}, "Region configuration differs"},
 		// 4
-		{&serverConfig{Browser: false}, &serverConfig{Browser: true}, "Browser configuration differs"},
-		// 5
-		{&serverConfig{Domain: "domain1"}, &serverConfig{Domain: "domain2"}, "Domain configuration differs"},
-		// 6
 		{
 			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 6}}},
 			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 4}}},
 			"StorageClass configuration differs",
 		},
-		// 7
+		// 5
 		{
 			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: false}}}},
 			"AMQP Notification configuration differs",
 		},
-		// 8
+		// 6
 		{
 			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: false}}}},
 			"NATS Notification configuration differs",
 		},
-		// 9
+		// 7
+		{
+			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: true}}}},
+			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: false}}}},
+			"NSQ Notification configuration differs",
+		},
+		// 8
 		{
 			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: false}}}},
 			"ElasticSearch Notification configuration differs",
 		},
-		// 10
+		// 9
 		{
 			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: false}}}},
 			"Redis Notification configuration differs",
 		},
-		// 11
+		// 10
 		{
 			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: false}}}},
 			"PostgreSQL Notification configuration differs",
 		},
-		// 12
+		// 11
 		{
 			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: false}}}},
 			"Kafka Notification configuration differs",
 		},
-		// 13
+		// 12
 		{
 			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: false}}}},
 			"Webhook Notification configuration differs",
 		},
-		// 14
+		// 13
 		{
 			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: false}}}},
 			"MySQL Notification configuration differs",
 		},
-		// 15
+		// 14
 		{
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
 			"MQTT Notification configuration differs",
 		},
+		// 15
+		{
+			&serverConfig{Logger: loggerConfig{
+				Console: loggerConsole{Enabled: true},
+				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address1"}},
+			}},
+			&serverConfig{Logger: loggerConfig{
+				Console: loggerConsole{Enabled: true},
+				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address2"}},
+			}},
+			"Logger configuration differs",
+		},
 	}
 
 	for i, testCase := range testCases {

cmd/config-dir.go

@@ -28,9 +28,6 @@ const (
 	// Default minio configuration directory where below configuration files/directories are stored.
 	defaultMinioConfigDir = ".minio"
 
-	// Minio configuration file.
-	minioConfigFile = "config.json"
-
 	// Directory contains below files/directories for HTTPS configuration.
 	certsDir = "certs"
 
@@ -77,7 +74,15 @@ func (config *ConfigDir) GetCADir() string {
 
 // Create - creates configuration directory tree.
 func (config *ConfigDir) Create() error {
-	return os.MkdirAll(config.GetCADir(), 0700)
+	err := os.MkdirAll(config.GetCADir(), 0700)
+	// It is possible in kubernetes like deployments this directory
+	// is already mounted and is not writable, ignore any write errors.
+	if err != nil {
+		if os.IsPermission(err) {
+			err = nil
+		}
+	}
+	return err
 }
 
 // GetMinioConfigFile - returns absolute path of config.json file.

cmd/config-migrate_test.go

@@ -25,15 +25,26 @@ import (
 
 // Test if config v1 is purged
 func TestServerConfigMigrateV1(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	objLayer, fsDir, err := prepareFS()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+	err = newTestConfig(globalMinioDefaultRegion, objLayer)
 	if err != nil {
 		t.Fatalf("Init Test config failed")
 	}
-	// remove the root directory after the test ends.
+	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
+	if err != nil {
+		t.Fatal(err)
+	}
 	defer os.RemoveAll(rootPath)
-
 	setConfigDir(rootPath)
 
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = objLayer
+	globalObjLayerMutex.Unlock()
+
 	// Create a V1 config json file and store it
 	configJSON := "{ \"version\":\"1\", \"accessKeyId\":\"abcde\", \"secretAccessKey\":\"abcdefgh\"}"
 	configPath := rootPath + "/fsUsers.json"
@@ -45,13 +56,14 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	if err := migrateConfig(); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
+
 	// Check if config v1 is removed from filesystem
 	if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
 		t.Fatal("Config V1 file is not purged")
 	}
 
 	// Initialize server config and check again if everything is fine
-	if err := loadConfig(); err != nil {
+	if err := loadConfig(objLayer); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 }
@@ -59,20 +71,13 @@ func TestServerConfigMigrateV1(t *testing.T) {
 // Test if all migrate code returns nil when config file does not
 // exist
 func TestServerConfigMigrateInexistentConfig(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
 	if err != nil {
-		t.Fatalf("Init Test config failed")
+		t.Fatal(err)
 	}
-	// remove the root directory after the test ends.
 	defer os.RemoveAll(rootPath)
 
 	setConfigDir(rootPath)
-	configPath := rootPath + "/" + minioConfigFile
-
-	// Remove config file
-	if err := os.Remove(configPath); err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
 
 	if err := migrateV2ToV3(); err != nil {
 		t.Fatal("migrate v2 to v3 should succeed when no config file is found")
@@ -134,18 +139,41 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	if err := migrateV21ToV22(); err != nil {
 		t.Fatal("migrate v21 to v22 should succeed when no config file is found")
 	}
+	if err := migrateV22ToV23(); err != nil {
+		t.Fatal("migrate v22 to v23 should succeed when no config file is found")
+	}
+	if err := migrateV23ToV24(); err != nil {
+		t.Fatal("migrate v23 to v24 should succeed when no config file is found")
+	}
+	if err := migrateV24ToV25(); err != nil {
+		t.Fatal("migrate v24 to v25 should succeed when no config file is found")
+	}
+	if err := migrateV25ToV26(); err != nil {
+		t.Fatal("migrate v25 to v26 should succeed when no config file is found")
+	}
+	if err := migrateV26ToV27(); err != nil {
+		t.Fatal("migrate v26 to v27 should succeed when no config file is found")
+	}
+	if err := migrateV27ToV28(); err != nil {
+		t.Fatal("migrate v27 to v28 should succeed when no config file is found")
+	}
 }
 
-// Test if a config migration from v2 to v23 is successfully done
-func TestServerConfigMigrateV2toV23(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+// Test if a config migration from v2 to v30 is successfully done
+func TestServerConfigMigrateV2toV30(t *testing.T) {
+	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
 	if err != nil {
-		t.Fatalf("Init Test config failed")
+		t.Fatal(err)
 	}
-	// remove the root directory after the test ends.
 	defer os.RemoveAll(rootPath)
-
 	setConfigDir(rootPath)
+
+	objLayer, fsDir, err := prepareFS()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+
 	configPath := rootPath + "/" + minioConfigFile
 
 	// Create a corrupted config file
@@ -171,8 +199,16 @@ func TestServerConfigMigrateV2toV23(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
+	if err := migrateConfigToMinioSys(objLayer); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
+	if err := migrateMinioSysConfig(objLayer); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
 	// Initialize server config and check again if everything is fine
-	if err := loadConfig(); err != nil {
+	if err := loadConfig(objLayer); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 
@@ -186,6 +222,7 @@ func TestServerConfigMigrateV2toV23(t *testing.T) {
 	if globalServerConfig.Credential.AccessKey != accessKey {
 		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, globalServerConfig.Credential.AccessKey)
 	}
+
 	if globalServerConfig.Credential.SecretKey != secretKey {
 		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, globalServerConfig.Credential.SecretKey)
 	}
@@ -193,13 +230,11 @@ func TestServerConfigMigrateV2toV23(t *testing.T) {
 
 // Test if all migrate code returns error with corrupted config files
 func TestServerConfigMigrateFaultyConfig(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
 	if err != nil {
-		t.Fatalf("Init Test config failed")
+		t.Fatal(err)
 	}
-	// remove the root directory after the test ends.
 	defer os.RemoveAll(rootPath)
-
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
 
@@ -272,17 +307,30 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV22ToV23(); err == nil {
 		t.Fatal("migrateConfigV22ToV23() should fail with a corrupted json")
 	}
+	if err := migrateV23ToV24(); err == nil {
+		t.Fatal("migrateConfigV23ToV24() should fail with a corrupted json")
+	}
+	if err := migrateV24ToV25(); err == nil {
+		t.Fatal("migrateConfigV24ToV25() should fail with a corrupted json")
+	}
+	if err := migrateV25ToV26(); err == nil {
+		t.Fatal("migrateConfigV25ToV26() should fail with a corrupted json")
+	}
+	if err := migrateV26ToV27(); err == nil {
+		t.Fatal("migrateConfigV26ToV27() should fail with a corrupted json")
+	}
+	if err := migrateV27ToV28(); err == nil {
+		t.Fatal("migrateConfigV27ToV28() should fail with a corrupted json")
+	}
 }
 
 // Test if all migrate code returns error with corrupted config files
 func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
 	if err != nil {
-		t.Fatalf("Init Test config failed")
+		t.Fatal(err)
 	}
-	// remove the root directory after the test ends.
 	defer os.RemoveAll(rootPath)
-
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
 

cmd/config-versions.go

@@ -19,8 +19,12 @@ package cmd
 import (
 	"sync"
 
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
+	"github.com/minio/minio/pkg/quick"
 )
 
 /////////////////// Config V1 ///////////////////
@@ -369,7 +373,7 @@ type serverConfigV12 struct {
 	Notify notifierV2 `json:"notify"`
 }
 
-type notifier struct {
+type notifierV3 struct {
 	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
 	Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
 	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
@@ -394,7 +398,7 @@ type serverConfigV13 struct {
 	Logger *loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV14 server configuration version '14' which is like
@@ -411,7 +415,7 @@ type serverConfigV14 struct {
 	Logger *loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV15 server configuration version '15' which is like
@@ -428,7 +432,7 @@ type serverConfigV15 struct {
 	Logger *loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // FileLogger is introduced to workaround the dependency about logrus
@@ -466,7 +470,7 @@ type serverConfigV16 struct {
 	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV17 server configuration version '17' which is like
@@ -485,7 +489,7 @@ type serverConfigV17 struct {
 	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV18 server configuration version '18' which is like
@@ -504,7 +508,7 @@ type serverConfigV18 struct {
 	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV19 server configuration version '19' which is like
@@ -522,7 +526,7 @@ type serverConfigV19 struct {
 	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV20 server configuration version '20' which is like
@@ -541,7 +545,7 @@ type serverConfigV20 struct {
 	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV21 is just like version '20' without logger field
@@ -556,7 +560,7 @@ type serverConfigV21 struct {
 	Domain     string           `json:"domain"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV22 is just like version '21' with added support
@@ -577,7 +581,7 @@ type serverConfigV22 struct {
 	StorageClass storageClassConfig `json:"storageclass"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV23 is just like version '22' with addition of cache field.
@@ -600,7 +604,7 @@ type serverConfigV23 struct {
 	Cache CacheConfig `json:"cache"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV24 is just like version '23', we had to revert
@@ -624,7 +628,7 @@ type serverConfigV24 struct {
 	Cache CacheConfig `json:"cache"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV25 is just like version '24', stores additionally
@@ -633,6 +637,8 @@ type serverConfigV24 struct {
 // IMPORTANT NOTE: When updating this struct make sure that
 // serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV25 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
 	Version string `json:"version"`
 
 	// S3 API configuration.
@@ -649,5 +655,240 @@ type serverConfigV25 struct {
 	Cache CacheConfig `json:"cache"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
+}
+
+// serverConfigV26 is just like version '25', stores additionally
+// cache max use value in 'CacheConfig'.
+type serverConfigV26 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifierV3 `json:"notify"`
+}
+
+type loggerConsole struct {
+	Enabled bool `json:"enabled"`
+}
+
+type loggerHTTP struct {
+	Enabled  bool   `json:"enabled"`
+	Endpoint string `json:"endpoint"`
+}
+
+type loggerConfig struct {
+	Console loggerConsole         `json:"console"`
+	HTTP    map[string]loggerHTTP `json:"http"`
+}
+
+// serverConfigV27 is just like version '26', stores additionally
+// the logger field
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV27 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifierV3 `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+}
+
+// serverConfigV28 is just like version '27', additionally
+// storing KMS config
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV28 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Worm       BoolFlag         `json:"worm"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// KMS configuration
+	KMS crypto.KMSConfig `json:"kms"`
+
+	// Notification queue configuration.
+	Notify notifierV3 `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+}
+
+// serverConfigV29 is just like version '28'.
+type serverConfigV29 serverConfigV28
+
+// compressionConfig represents the compression settings.
+type compressionConfig struct {
+	Enabled    bool     `json:"enabled"`
+	Extensions []string `json:"extensions"`
+	MimeTypes  []string `json:"mime-types"`
+}
+
+// serverConfigV30 is just like version '29', stores additionally
+// extensions and mimetypes fields for compression.
+type serverConfigV30 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Worm       BoolFlag         `json:"worm"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// KMS configuration
+	KMS crypto.KMSConfig `json:"kms"`
+
+	// Notification queue configuration.
+	Notify notifierV3 `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+
+	// Compression configuration
+	Compression compressionConfig `json:"compress"`
+}
+
+// serverConfigV31 is just like version '30', with OPA and OpenID configuration.
+type serverConfigV31 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Worm       BoolFlag         `json:"worm"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// KMS configuration
+	KMS crypto.KMSConfig `json:"kms"`
+
+	// Notification queue configuration.
+	Notify notifierV3 `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+
+	// Compression configuration
+	Compression compressionConfig `json:"compress"`
+
+	// OpenID configuration
+	OpenID struct {
+		// JWKS validator config.
+		JWKS validator.JWKSArgs `json:"jwks"`
+	} `json:"openid"`
+
+	// External policy enforcements.
+	Policy struct {
+		// OPA configuration.
+		OPA iampolicy.OpaArgs `json:"opa"`
+
+		// Add new external policy enforcements here.
+	} `json:"policy"`
+}
+
+type notifier struct {
+	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
+	Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
+	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
+	MQTT          map[string]target.MQTTArgs          `json:"mqtt"`
+	MySQL         map[string]target.MySQLArgs         `json:"mysql"`
+	NATS          map[string]target.NATSArgs          `json:"nats"`
+	NSQ           map[string]target.NSQArgs           `json:"nsq"`
+	PostgreSQL    map[string]target.PostgreSQLArgs    `json:"postgresql"`
+	Redis         map[string]target.RedisArgs         `json:"redis"`
+	Webhook       map[string]target.WebhookArgs       `json:"webhook"`
+}
+
+// serverConfigV32 is just like version '31' with added nsq notifer.
+type serverConfigV32 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Worm       BoolFlag         `json:"worm"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// KMS configuration
+	KMS crypto.KMSConfig `json:"kms"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+
+	// Compression configuration
+	Compression compressionConfig `json:"compress"`
+
+	// OpenID configuration
+	OpenID struct {
+		// JWKS validator config.
+		JWKS validator.JWKSArgs `json:"jwks"`
+	} `json:"openid"`
+
+	// External policy enforcements.
+	Policy struct {
+		// OPA configuration.
+		OPA iampolicy.OpaArgs `json:"opa"`
+
+		// Add new external policy enforcements here.
+	} `json:"policy"`
 }

cmd/config.go

@@ -0,0 +1,214 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"os"
+	"path"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/quick"
+)
+
+const (
+	minioConfigPrefix = "config"
+
+	// Minio configuration file.
+	minioConfigFile = "config.json"
+
+	// Minio backup file
+	minioConfigBackupFile = minioConfigFile + ".backup"
+)
+
+func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverConfig) error {
+	if err := quick.CheckData(config); err != nil {
+		return err
+	}
+
+	data, err := json.MarshalIndent(config, "", "\t")
+	if err != nil {
+		return err
+	}
+
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+	if globalEtcdClient != nil {
+		timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+		_, err = globalEtcdClient.Put(timeoutCtx, configFile, string(data))
+		defer cancel()
+		return err
+	}
+
+	// Create a backup of the current config
+	oldData, err := readConfig(ctx, objAPI, configFile)
+	if err == nil {
+		backupConfigFile := path.Join(minioConfigPrefix, minioConfigBackupFile)
+		if err = saveConfig(ctx, objAPI, backupConfigFile, oldData); err != nil {
+			return err
+		}
+	} else {
+		if err != errConfigNotFound {
+			return err
+		}
+	}
+
+	// Save the new config in the std config path
+	return saveConfig(ctx, objAPI, configFile, data)
+}
+
+func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
+	var configData []byte
+	var err error
+
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+	if globalEtcdClient != nil {
+		configData, err = readConfigEtcd(ctx, globalEtcdClient, configFile)
+	} else {
+		configData, err = readConfig(ctx, objAPI, configFile)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if runtime.GOOS == "windows" {
+		configData = bytes.Replace(configData, []byte("\r\n"), []byte("\n"), -1)
+	}
+
+	if err = quick.CheckDuplicateKeys(string(configData)); err != nil {
+		return nil, err
+	}
+
+	var config = &serverConfig{}
+	if err = json.Unmarshal(configData, config); err != nil {
+		return nil, err
+	}
+
+	if err = quick.CheckData(config); err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}
+
+// ConfigSys - config system.
+type ConfigSys struct{}
+
+// Load - load config.json.
+func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
+	return sys.Init(objAPI)
+}
+
+// Init - initializes config system from config.json.
+func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Initializing configuration needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	//  - Write quorum not met when upgrading configuration
+	//    version is needed.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			err := initConfig(objAPI)
+			if err != nil {
+				if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for configuration to be initialized..")
+					continue
+				}
+				return err
+			}
+
+			return nil
+		}
+	}
+}
+
+// NewConfigSys - creates new config system object.
+func NewConfigSys() *ConfigSys {
+	return &ConfigSys{}
+}
+
+// Initialize and load config from remote etcd or local config directory
+func initConfig(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	if globalEtcdClient != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
+		cancel()
+		if err == nil && resp.Count > 0 {
+			if err = migrateConfig(); err != nil {
+				return err
+			}
+
+			// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
+			if err := migrateConfigToMinioSys(objAPI); err != nil {
+				return err
+			}
+		}
+	} else {
+		if isFile(getConfigFile()) {
+			if err := migrateConfig(); err != nil {
+				return err
+			}
+		}
+		// Migrates ${HOME}/.minio/config.json or config.json.deprecated
+		// to '<export_path>/.minio.sys/config/config.json'
+		// ignore if the file doesn't exist.
+		if err := migrateConfigToMinioSys(objAPI); err != nil && !os.IsNotExist(err) {
+			return err
+		}
+	}
+
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	// Watch config for changes and reloads them in-memory.
+	go watchConfig(objAPI, configFile, loadConfig)
+
+	if err := checkConfig(context.Background(), objAPI, configFile); err != nil {
+		if err == errConfigNotFound {
+			// Config file does not exist, we create it fresh and return upon success.
+			if err = newSrvConfig(objAPI); err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	}
+
+	if err := migrateMinioSysConfig(objAPI); err != nil {
+		return err
+	}
+
+	return loadConfig(objAPI)
+}

cmd/copy-part-range.go

@@ -17,90 +17,52 @@
 package cmd
 
 import (
-	"fmt"
 	"net/http"
 	"net/url"
-	"strconv"
-	"strings"
 )
 
 // Writes S3 compatible copy part range error.
-func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL) {
+func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL, browser bool) {
 	switch err {
 	case errInvalidRange:
-		writeErrorResponse(w, ErrInvalidCopyPartRange, url)
+		writeErrorResponse(w, ErrInvalidCopyPartRange, url, browser)
 		return
 	case errInvalidRangeSource:
-		writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url)
+		writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url, browser)
 		return
 	default:
-		writeErrorResponse(w, ErrInternalError, url)
+		writeErrorResponse(w, ErrInternalError, url, browser)
 		return
 	}
 }
 
-// Parses x-amz-copy-source-range for CopyObjectPart API. Specifically written to
-// differentiate the behavior between regular httpRange header v/s x-amz-copy-source-range.
-// The range of bytes to copy from the source object. The range value must use the form
-// bytes=first-last, where the first and last are the zero-based byte offsets to copy.
-// For example, bytes=0-9 indicates that you want to copy the first ten bytes of the source.
+// Parses x-amz-copy-source-range for CopyObjectPart API. Its behavior
+// is different from regular HTTP range header. It only supports the
+// form `bytes=first-last` where first and last are zero-based byte
+// offsets. See
 // http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html
-func parseCopyPartRange(rangeString string, resourceSize int64) (hrange *httpRange, err error) {
-	// Return error if given range string doesn't start with byte range prefix.
-	if !strings.HasPrefix(rangeString, byteRangePrefix) {
-		return nil, fmt.Errorf("'%s' does not start with '%s'", rangeString, byteRangePrefix)
+// for full details. This function treats an empty rangeString as
+// referring to the whole resource.
+func parseCopyPartRangeSpec(rangeString string) (hrange *HTTPRangeSpec, err error) {
+	hrange, err = parseRequestRangeSpec(rangeString)
+	if err != nil {
+		return nil, err
 	}
-
-	// Trim byte range prefix.
-	byteRangeString := strings.TrimPrefix(rangeString, byteRangePrefix)
-
-	// Check if range string contains delimiter '-', else return error. eg. "bytes=8"
-	sepIndex := strings.Index(byteRangeString, "-")
-	if sepIndex == -1 {
+	if hrange.IsSuffixLength || hrange.Start < 0 || hrange.End < 0 {
 		return nil, errInvalidRange
 	}
-
-	offsetBeginString := byteRangeString[:sepIndex]
-	offsetBegin := int64(-1)
-	// Convert offsetBeginString only if its not empty.
-	if len(offsetBeginString) > 0 {
-		if !validBytePos.MatchString(offsetBeginString) {
-			return nil, errInvalidRange
-		}
-		if offsetBegin, err = strconv.ParseInt(offsetBeginString, 10, 64); err != nil {
-			return nil, errInvalidRange
-		}
-	}
-
-	offsetEndString := byteRangeString[sepIndex+1:]
-	offsetEnd := int64(-1)
-	// Convert offsetEndString only if its not empty.
-	if len(offsetEndString) > 0 {
-		if !validBytePos.MatchString(offsetEndString) {
-			return nil, errInvalidRange
-		}
-		if offsetEnd, err = strconv.ParseInt(offsetEndString, 10, 64); err != nil {
-			return nil, errInvalidRange
-		}
-	}
-
-	// rangeString contains first byte positions. eg. "bytes=2-" or
-	// rangeString contains last bye positions. eg. "bytes=-2"
-	if offsetBegin == -1 || offsetEnd == -1 {
-		return nil, errInvalidRange
-	}
-
-	// Last byte position should not be greater than first byte
-	// position. eg. "bytes=5-2"
-	if offsetBegin > offsetEnd {
-		return nil, errInvalidRange
-	}
-
-	// First and last byte positions should not be >= resourceSize.
-	if offsetBegin >= resourceSize || offsetEnd >= resourceSize {
-		return nil, errInvalidRangeSource
-	}
-
-	// Success..
-	return &httpRange{offsetBegin, offsetEnd, resourceSize}, nil
+	return hrange, nil
+}
+
+// checkCopyPartRangeWithSize adds more check to the range string in case of
+// copy object part. This API requires having specific start and end  range values
+// e.g. 'bytes=3-10'. Other use cases will be rejected.
+func checkCopyPartRangeWithSize(rs *HTTPRangeSpec, resourceSize int64) (err error) {
+	if rs == nil {
+		return nil
+	}
+	if rs.IsSuffixLength || rs.Start >= resourceSize || rs.End >= resourceSize {
+		return errInvalidRangeSource
+	}
+	return nil
 }

cmd/copy-part-range_test.go

@@ -19,35 +19,37 @@ package cmd
 import "testing"
 
 // Test parseCopyPartRange()
-func TestParseCopyPartRange(t *testing.T) {
+func TestParseCopyPartRangeSpec(t *testing.T) {
 	// Test success cases.
 	successCases := []struct {
 		rangeString string
 		offsetBegin int64
 		offsetEnd   int64
-		length      int64
 	}{
-		{"bytes=2-5", 2, 5, 4},
-		{"bytes=2-9", 2, 9, 8},
-		{"bytes=2-2", 2, 2, 1},
-		{"bytes=0000-0006", 0, 6, 7},
+		{"bytes=2-5", 2, 5},
+		{"bytes=2-9", 2, 9},
+		{"bytes=2-2", 2, 2},
+		{"bytes=0000-0006", 0, 6},
 	}
+	objectSize := int64(10)
 
 	for _, successCase := range successCases {
-		hrange, err := parseCopyPartRange(successCase.rangeString, 10)
+		rs, err := parseCopyPartRangeSpec(successCase.rangeString)
 		if err != nil {
 			t.Fatalf("expected: <nil>, got: %s", err)
 		}
 
-		if hrange.offsetBegin != successCase.offsetBegin {
-			t.Fatalf("expected: %d, got: %d", successCase.offsetBegin, hrange.offsetBegin)
+		start, length, err1 := rs.GetOffsetLength(objectSize)
+		if err1 != nil {
+			t.Fatalf("expected: <nil>, got: %s", err1)
 		}
 
-		if hrange.offsetEnd != successCase.offsetEnd {
-			t.Fatalf("expected: %d, got: %d", successCase.offsetEnd, hrange.offsetEnd)
+		if start != successCase.offsetBegin {
+			t.Fatalf("expected: %d, got: %d", successCase.offsetBegin, start)
 		}
-		if hrange.getLength() != successCase.length {
-			t.Fatalf("expected: %d, got: %d", successCase.length, hrange.getLength())
+
+		if start+length-1 != successCase.offsetEnd {
+			t.Fatalf("expected: %d, got: %d", successCase.offsetEnd, start+length-1)
 		}
 	}
 
@@ -59,15 +61,16 @@ func TestParseCopyPartRange(t *testing.T) {
 		"bytes=2-+5",
 		"bytes=2--5",
 		"bytes=-",
-		"",
 		"2-5",
 		"bytes = 2-5",
 		"bytes=2 - 5",
 		"bytes=0-0,-1",
 		"bytes=2-5 ",
+		"bytes=-1",
+		"bytes=1-",
 	}
 	for _, rangeString := range invalidRangeStrings {
-		if _, err := parseCopyPartRange(rangeString, 10); err == nil {
+		if _, err := parseCopyPartRangeSpec(rangeString); err == nil {
 			t.Fatalf("expected: an error, got: <nil> for range %s", rangeString)
 		}
 	}
@@ -78,8 +81,14 @@ func TestParseCopyPartRange(t *testing.T) {
 		"bytes=20-30",
 	}
 	for _, rangeString := range errorRangeString {
-		if _, err := parseCopyPartRange(rangeString, 10); err != errInvalidRangeSource {
-			t.Fatalf("expected: %s, got: %s", errInvalidRangeSource, err)
+		rs, err := parseCopyPartRangeSpec(rangeString)
+		if err == nil {
+			err1 := checkCopyPartRangeWithSize(rs, objectSize)
+			if err1 != errInvalidRangeSource {
+				t.Fatalf("expected: %s, got: %s", errInvalidRangeSource, err)
+			}
+		} else {
+			t.Fatalf("expected: %s, got: <nil>", errInvalidRangeSource)
 		}
 	}
 }

cmd/crypto/config.go

@@ -0,0 +1,20 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+// KMSConfig has the KMS config for hashicorp vault
+type KMSConfig struct {
+	Vault VaultConfig `json:"vault"`
+}

cmd/crypto/doc.go

@@ -0,0 +1,116 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package crypto implements AWS S3 related cryptographic building blocks
+// for implementing Server-Side-Encryption (SSE-S3) and Server-Side-Encryption
+// with customer provided keys (SSE-C).
+//
+// All objects are encrypted with an unique and randomly generated 'ObjectKey'.
+// The ObjectKey itself is never stored in plaintext. Instead it is only stored
+// in a sealed from. The sealed 'ObjectKey' is created by encrypting the 'ObjectKey'
+// with an unique key-encryption-key. Given the correct key-encryption-key the
+// sealed 'ObjectKey' can be unsealed and the object can be decrypted.
+//
+//
+// ## SSE-C
+//
+// SSE-C computes the key-encryption-key from the client-provided key, an
+// initialization vector (IV) and the bucket/object path.
+//
+// 1. Encrypt:
+//       Input: ClientKey, bucket, object, metadata, object_data
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: ClientKey, bucket, object, metadata, enc_object_data
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -   KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+//
+// ## SSE-S3
+//
+// SSE-S3 can use either a master key or a KMS as root-of-trust.
+// The en/decryption slightly depens upon which root-of-trust is used.
+//
+// ### SSE-S3 and single master key
+//
+// The master key is used to derive unique object- and key-encryption-keys.
+// SSE-S3 with a single master key works as SSE-C where the master key is
+// used as the client-provided key.
+//
+// 1. Encrypt:
+//       Input: MasterKey, bucket, object, metadata, object_data
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: MasterKey, bucket, object, metadata, enc_object_data
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -   KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+//
+// ### SSE-S3 and KMS
+//
+// SSE-S3 requires that the KMS provides two functions:
+//       1.       Generate(KeyID) -> (Key, EncKey)
+//       2. Unseal(KeyID, EncKey) -> Key
+//
+// 1. Encrypt:
+//       Input: KeyID, bucket, object, metadata, object_data
+//       -     Key, EncKey := Generate(KeyID)
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(Key, Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- KeyID
+//       -        metadata <- EncKey
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: bucket, object, metadata, enc_object_data
+//       -      KeyID  <- metadata
+//       -      EncKey <- metadata
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -         Key := Unseal(KeyID, EncKey)
+//       -   KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+package crypto

cmd/crypto/error.go

@@ -0,0 +1,71 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import "errors"
+
+// Error is the generic type for any error happening during decrypting
+// an object. It indicates that the object itself or its metadata was
+// modified accidentally or maliciously.
+type Error struct{ msg string }
+
+func (e Error) Error() string { return e.msg }
+
+var (
+	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
+	// is not supported.
+	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+
+	// ErrInvalidCustomerAlgorithm indicates that the specified SSE-C algorithm
+	// is not supported.
+	ErrInvalidCustomerAlgorithm = errors.New("The SSE-C algorithm is not supported")
+
+	// ErrMissingCustomerKey indicates that the HTTP headers contains no SSE-C client key.
+	ErrMissingCustomerKey = errors.New("The SSE-C request is missing the customer key")
+
+	// ErrMissingCustomerKeyMD5 indicates that the HTTP headers contains no SSE-C client key
+	// MD5 checksum.
+	ErrMissingCustomerKeyMD5 = errors.New("The SSE-C request is missing the customer key MD5")
+
+	// ErrInvalidCustomerKey indicates that the SSE-C client key is not valid - e.g. not a
+	// base64-encoded string or not 256 bits long.
+	ErrInvalidCustomerKey = errors.New("The SSE-C client key is invalid")
+
+	// ErrSecretKeyMismatch indicates that the provided secret key (SSE-C client key / SSE-S3 KMS key)
+	// does not match the secret key used during encrypting the object.
+	ErrSecretKeyMismatch = errors.New("The secret key does not match the secret key used during upload")
+
+	// ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the
+	// computed MD5 sum. This means that the client provided either the wrong key for
+	// a certain MD5 checksum or the wrong MD5 for a certain key.
+	ErrCustomerKeyMD5Mismatch = errors.New("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
+	// ErrIncompatibleEncryptionMethod indicates that both SSE-C headers and SSE-S3 headers were specified, and are incompatible
+	// The client needs to remove the SSE-S3 header or the SSE-C headers
+	ErrIncompatibleEncryptionMethod = errors.New("Server side encryption specified with both SSE-C and SSE-S3 headers")
+)
+
+var (
+	errMissingInternalIV            = Error{"The object metadata is missing the internal encryption IV"}
+	errMissingInternalSealAlgorithm = Error{"The object metadata is missing the internal seal algorithm"}
+
+	errInvalidInternalIV            = Error{"The internal encryption IV is malformed"}
+	errInvalidInternalSealAlgorithm = Error{"The internal seal algorithm is invalid and not supported"}
+)
+
+var (
+	// errOutOfEntropy indicates that the a source of randomness (PRNG) wasn't able
+	// to produce enough random data. This is fatal error and should cause a panic.
+	errOutOfEntropy = errors.New("Unable to read enough randomness from the system")
+)

cmd/crypto/header.go

@@ -0,0 +1,220 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"net/http"
+	"strings"
+)
+
+// SSEHeader is the general AWS SSE HTTP header key.
+const SSEHeader = "X-Amz-Server-Side-Encryption"
+
+const (
+	// SSEKmsID is the HTTP header key referencing the SSE-KMS
+	// key ID.
+	SSEKmsID = SSEHeader + "-Aws-Kms-Key-Id"
+
+	// SSEKmsContext is the HTTP header key referencing the
+	// SSE-KMS encryption context.
+	SSEKmsContext = SSEHeader + "-Context"
+)
+
+const (
+	// SSECAlgorithm is the HTTP header key referencing
+	// the SSE-C algorithm.
+	SSECAlgorithm = SSEHeader + "-Customer-Algorithm"
+
+	// SSECKey is the HTTP header key referencing the
+	// SSE-C client-provided key..
+	SSECKey = SSEHeader + "-Customer-Key"
+
+	// SSECKeyMD5 is the HTTP header key referencing
+	// the MD5 sum of the client-provided key.
+	SSECKeyMD5 = SSEHeader + "-Customer-Key-Md5"
+)
+
+const (
+	// SSECopyAlgorithm is the HTTP header key referencing
+	// the SSE-C algorithm for SSE-C copy requests.
+	SSECopyAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
+
+	// SSECopyKey is the HTTP header key referencing the SSE-C
+	// client-provided key for SSE-C copy requests.
+	SSECopyKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
+
+	// SSECopyKeyMD5 is the HTTP header key referencing the
+	// MD5 sum of the client key for SSE-C copy requests.
+	SSECopyKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5"
+)
+
+const (
+	// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
+	// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
+	// For SSE-C  see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
+	SSEAlgorithmAES256 = "AES256"
+
+	// SSEAlgorithmKMS is the value of 'X-Amz-Server-Side-Encryption' for SSE-KMS.
+	// See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+	SSEAlgorithmKMS = "aws:kms"
+)
+
+// RemoveSensitiveHeaders removes confidential encryption
+// information - e.g. the SSE-C key - from the HTTP headers.
+// It has the same semantics as RemoveSensitiveEntires.
+func RemoveSensitiveHeaders(h http.Header) {
+	h.Del(SSECKey)
+	h.Del(SSECopyKey)
+}
+
+// S3 represents AWS SSE-S3. It provides functionality to handle
+// SSE-S3 requests.
+var S3 = s3{}
+
+type s3 struct{}
+
+// IsRequested returns true if the HTTP headers indicates that
+// the S3 client requests SSE-S3.
+func (s3) IsRequested(h http.Header) bool {
+	_, ok := h[SSEHeader]
+	return ok && strings.ToLower(h.Get(SSEHeader)) != SSEAlgorithmKMS // Return only true if the SSE header is specified and does not contain the SSE-KMS value
+}
+
+// ParseHTTP parses the SSE-S3 related HTTP headers and checks
+// whether they contain valid values.
+func (s3) ParseHTTP(h http.Header) (err error) {
+	if h.Get(SSEHeader) != SSEAlgorithmAES256 {
+		err = ErrInvalidEncryptionMethod
+	}
+	return
+}
+
+// S3KMS represents AWS SSE-KMS. It provides functionality to
+// handle SSE-KMS requests.
+var S3KMS = s3KMS{}
+
+type s3KMS struct{}
+
+// IsRequested returns true if the HTTP headers indicates that
+// the S3 client requests SSE-KMS.
+func (s3KMS) IsRequested(h http.Header) bool {
+	if _, ok := h[SSEKmsID]; ok {
+		return true
+	}
+	if _, ok := h[SSEKmsContext]; ok {
+		return true
+	}
+	if _, ok := h[SSEHeader]; ok {
+		return strings.ToUpper(h.Get(SSEHeader)) != SSEAlgorithmAES256 // Return only true if the SSE header is specified and does not contain the SSE-S3 value
+	}
+	return false
+}
+
+var (
+	// SSEC represents AWS SSE-C. It provides functionality to handle
+	// SSE-C requests.
+	SSEC = ssec{}
+
+	// SSECopy represents AWS SSE-C for copy requests. It provides
+	// functionality to handle SSE-C copy requests.
+	SSECopy = ssecCopy{}
+)
+
+type ssec struct{}
+type ssecCopy struct{}
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C header. SSE-C copy headers are ignored.
+func (ssec) IsRequested(h http.Header) bool {
+	if _, ok := h[SSECAlgorithm]; ok {
+		return true
+	}
+	if _, ok := h[SSECKey]; ok {
+		return true
+	}
+	if _, ok := h[SSECKeyMD5]; ok {
+		return true
+	}
+	return false
+}
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C copy header. Regular SSE-C headers
+// are ignored.
+func (ssecCopy) IsRequested(h http.Header) bool {
+	if _, ok := h[SSECopyAlgorithm]; ok {
+		return true
+	}
+	if _, ok := h[SSECopyKey]; ok {
+		return true
+	}
+	if _, ok := h[SSECopyKeyMD5]; ok {
+		return true
+	}
+	return false
+}
+
+// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
+// on success. SSE-C copy headers are ignored.
+func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
+	if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
+		return key, ErrInvalidCustomerAlgorithm
+	}
+	if h.Get(SSECKey) == "" {
+		return key, ErrMissingCustomerKey
+	}
+	if h.Get(SSECKeyMD5) == "" {
+		return key, ErrMissingCustomerKeyMD5
+	}
+
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECKey))
+	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+		return key, ErrInvalidCustomerKey
+	}
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECKeyMD5))
+	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+		return key, ErrCustomerKeyMD5Mismatch
+	}
+	copy(key[:], clientKey)
+	return key, nil
+}
+
+// ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key
+// on success. Regular SSE-C headers are ignored.
+func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) {
+	if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
+		return key, ErrInvalidCustomerAlgorithm
+	}
+	if h.Get(SSECopyKey) == "" {
+		return key, ErrMissingCustomerKey
+	}
+	if h.Get(SSECopyKeyMD5) == "" {
+		return key, ErrMissingCustomerKeyMD5
+	}
+
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKey))
+	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+		return key, ErrInvalidCustomerKey
+	}
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKeyMD5))
+	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+		return key, ErrCustomerKeyMD5Mismatch
+	}
+	copy(key[:], clientKey)
+	return key, nil
+}

cmd/crypto/header_test.go

@@ -0,0 +1,436 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"net/http"
+	"sort"
+	"testing"
+)
+
+var kmsIsRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{}, Expected: false},                                                                                     // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"aws:kms"}}, Expected: true},                                   // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"0839-9047947-844842874-481"}}, Expected: true}, // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Context": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true},          // 3
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption":                []string{""},
+			"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{""},
+			"X-Amz-Server-Side-Encryption-Context":        []string{""},
+		},
+		Expected: true,
+	}, // 4
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption":                []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{""},
+		},
+		Expected: true,
+	}, // 5
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: false}, // 6
+}
+
+func TestKMSIsRequested(t *testing.T) {
+	for i, test := range kmsIsRequestedTests {
+		if got := S3KMS.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var s3IsRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},         // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true},        // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},               // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},         // 3
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{SSEAlgorithmKMS}}, Expected: false}, // 4
+}
+
+func TestS3IsRequested(t *testing.T) {
+	for i, test := range s3IsRequestedTests {
+		if got := S3.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var s3ParseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, ExpectedErr: nil},                         // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, ExpectedErr: ErrInvalidEncryptionMethod},        // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, ExpectedErr: ErrInvalidEncryptionMethod},   // 3
+}
+
+func TestS3Parse(t *testing.T) {
+	for i, test := range s3ParseTests {
+		if err := S3.ParseHTTP(test.Header); err != test.ExpectedErr {
+			t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+	}
+}
+
+var ssecIsRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{}, Expected: false},                                                                                                     // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true},                                 // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true},                 // 3
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{""},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{""},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{""},
+		},
+		Expected: true,
+	}, // 4
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: true,
+	}, // 5
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: false,
+	}, // 6
+}
+
+func TestSSECIsRequested(t *testing.T) {
+	for i, test := range ssecIsRequestedTests {
+		if got := SSEC.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var ssecCopyIsRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{}, Expected: false},                                                                                                                 // 0
+	{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true},                                 // 1
+	{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2
+	{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true},                 // 3
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{""},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{""},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{""},
+		},
+		Expected: true,
+	}, // 4
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: true,
+	}, // 5
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: false,
+	}, // 6
+}
+
+func TestSSECopyIsRequested(t *testing.T) {
+	for i, test := range ssecCopyIsRequestedTests {
+		if got := SSECopy.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var ssecParseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: nil, // 0
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES-256"}, // invalid algorithm
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerAlgorithm, // 1
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{""}, // no client key
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrMissingCustomerKey, // 2
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRr.ZXltdXN0cHJvdmlkZWQ="}, // invalid key
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerKey, // 3
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{""}, // no key MD5
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5, // 4
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"DzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, // wrong client key
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 5
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{".7PpPLAK26ONlVUGOWlusfg=="}, // wrong key MD5
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 6
+	},
+}
+
+func TestSSECParse(t *testing.T) {
+	var zeroKey [32]byte
+	for i, test := range ssecParseTests {
+		key, err := SSEC.ParseHTTP(test.Header)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: want error '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+
+		if err != nil && key != zeroKey {
+			t.Errorf("Test %d: parsing failed and client key is not zero key", i)
+		}
+		if err == nil && key == zeroKey {
+			t.Errorf("Test %d: parsed client key is zero key", i)
+		}
+	}
+}
+
+var ssecCopyParseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: nil, // 0
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES-256"}, // invalid algorithm
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerAlgorithm, // 1
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{""}, // no client key
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrMissingCustomerKey, // 2
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRr.ZXltdXN0cHJvdmlkZWQ="}, // invalid key
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerKey, // 3
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{""}, // no key MD5
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5, // 4
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"DzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, // wrong client key
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 5
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{".7PpPLAK26ONlVUGOWlusfg=="}, // wrong key MD5
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 6
+	},
+}
+
+func TestSSECopyParse(t *testing.T) {
+	var zeroKey [32]byte
+	for i, test := range ssecCopyParseTests {
+		key, err := SSECopy.ParseHTTP(test.Header)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: want error '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+
+		if err != nil && key != zeroKey {
+			t.Errorf("Test %d: parsing failed and client key is not zero key", i)
+		}
+		if err == nil && key == zeroKey {
+			t.Errorf("Test %d: parsed client key is zero key", i)
+		}
+		if _, ok := test.Header[SSECKey]; ok {
+			t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
+		}
+	}
+}
+
+var removeSensitiveHeadersTests = []struct {
+	Header, ExpectedHeader http.Header
+}{
+	{
+		Header: http.Header{
+			SSECKey:    []string{""},
+			SSECopyKey: []string{""},
+		},
+		ExpectedHeader: http.Header{},
+	},
+	{ // Standard SSE-C request headers
+		Header: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+	},
+	{ // Standard SSE-C + SSE-C-copy request headers
+		Header: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+	},
+	{ // Standard SSE-C + metadata request headers
+		Header: http.Header{
+			SSECAlgorithm:       []string{SSEAlgorithmAES256},
+			SSECKey:             []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm:       []string{SSEAlgorithmAES256},
+			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+	},
+}
+
+func TestRemoveSensitiveHeaders(t *testing.T) {
+	isEqual := func(x, y http.Header) bool {
+		if len(x) != len(y) {
+			return false
+		}
+		for k, v := range x {
+			u, ok := y[k]
+			if !ok || len(v) != len(u) {
+				return false
+			}
+			sort.Strings(v)
+			sort.Strings(u)
+			for j := range v {
+				if v[j] != u[j] {
+					return false
+				}
+			}
+		}
+		return true
+	}
+	areKeysEqual := func(h http.Header, metadata map[string]string) bool {
+		if len(h) != len(metadata) {
+			return false
+		}
+		for k := range h {
+			if _, ok := metadata[k]; !ok {
+				return false
+			}
+		}
+		return true
+	}
+
+	for i, test := range removeSensitiveHeadersTests {
+		metadata := make(map[string]string, len(test.Header))
+		for k := range test.Header {
+			metadata[k] = "" // set metadata key - we don't care about the value
+		}
+
+		RemoveSensitiveHeaders(test.Header)
+		if !isEqual(test.ExpectedHeader, test.Header) {
+			t.Errorf("Test %d: filtered headers do not match expected headers - got: %v , want: %v", i, test.Header, test.ExpectedHeader)
+		}
+		RemoveSensitiveEntries(metadata)
+		if !areKeysEqual(test.ExpectedHeader, metadata) {
+			t.Errorf("Test %d: filtered headers do not match expected headers - got: %v , want: %v", i, test.Header, test.ExpectedHeader)
+		}
+	}
+}

cmd/crypto/key.go

@@ -0,0 +1,176 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+// ObjectKey is a 256 bit secret key used to encrypt the object.
+// It must never be stored in plaintext.
+type ObjectKey [32]byte
+
+// GenerateKey generates a unique ObjectKey from a 256 bit external key
+// and a source of randomness. If random is nil the default PRNG of the
+// system (crypto/rand) is used.
+func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
+	if random == nil {
+		random = rand.Reader
+	}
+	var nonce [32]byte
+	if _, err := io.ReadFull(random, nonce[:]); err != nil {
+		logger.CriticalIf(context.Background(), errOutOfEntropy)
+	}
+	sha := sha256.New()
+	sha.Write(extKey[:])
+	sha.Write(nonce[:])
+	sha.Sum(key[:0])
+	return key
+}
+
+// GenerateIV generates a new random 256 bit IV from the provided source
+// of randomness. If random is nil the default PRNG of the system
+// (crypto/rand) is used.
+func GenerateIV(random io.Reader) (iv [32]byte) {
+	if random == nil {
+		random = rand.Reader
+	}
+	if _, err := io.ReadFull(random, iv[:]); err != nil {
+		logger.CriticalIf(context.Background(), errOutOfEntropy)
+	}
+	return iv
+}
+
+// SealedKey represents a sealed object key. It can be stored
+// at an untrusted location.
+type SealedKey struct {
+	Key       [64]byte // The encrypted and authenticted object-key.
+	IV        [32]byte // The random IV used to encrypt the object-key.
+	Algorithm string   // The sealing algorithm used to encrypt the object key.
+}
+
+// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
+// key is also cryptographically bound to the object's path (bucket/object) and the
+// domain (SSE-C or SSE-S3).
+func (key ObjectKey) Seal(extKey, iv [32]byte, domain, bucket, object string) SealedKey {
+	var (
+		sealingKey   [32]byte
+		encryptedKey bytes.Buffer
+	)
+	mac := hmac.New(sha256.New, extKey[:])
+	mac.Write(iv[:])
+	mac.Write([]byte(domain))
+	mac.Write([]byte(SealAlgorithm))
+	mac.Write([]byte(path.Join(bucket, object))) // use path.Join for canonical 'bucket/object'
+	mac.Sum(sealingKey[:0])
+	if n, err := sio.Encrypt(&encryptedKey, bytes.NewReader(key[:]), sio.Config{Key: sealingKey[:]}); n != 64 || err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
+	}
+	sealedKey := SealedKey{
+		IV:        iv,
+		Algorithm: SealAlgorithm,
+	}
+	copy(sealedKey.Key[:], encryptedKey.Bytes())
+	return sealedKey
+}
+
+// Unseal decrypts a sealed key using the 256 bit external key. Since the sealed key
+// may be cryptographically bound to the object's path the same bucket/object as during sealing
+// must be provided. On success the ObjectKey contains the decrypted sealed key.
+func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucket, object string) error {
+	var (
+		unsealConfig sio.Config
+		decryptedKey bytes.Buffer
+	)
+	switch sealedKey.Algorithm {
+	default:
+		return Error{fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm)}
+	case SealAlgorithm:
+		mac := hmac.New(sha256.New, extKey[:])
+		mac.Write(sealedKey.IV[:])
+		mac.Write([]byte(domain))
+		mac.Write([]byte(SealAlgorithm))
+		mac.Write([]byte(path.Join(bucket, object))) // use path.Join for canonical 'bucket/object'
+		unsealConfig = sio.Config{MinVersion: sio.Version20, Key: mac.Sum(nil)}
+	case InsecureSealAlgorithm:
+		sha := sha256.New()
+		sha.Write(extKey[:])
+		sha.Write(sealedKey.IV[:])
+		unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)}
+	}
+
+	if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
+		return ErrSecretKeyMismatch
+	}
+	copy(key[:], decryptedKey.Bytes())
+	return nil
+}
+
+// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index.
+func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
+	var bin [4]byte
+	binary.LittleEndian.PutUint32(bin[:], id)
+
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write(bin[:])
+	mac.Sum(partKey[:0])
+	return partKey
+}
+
+// SealETag seals the etag using the object key.
+// It does not encrypt empty ETags because such ETags indicate
+// that the S3 client hasn't sent an ETag = MD5(object) and
+// the backend can pick an ETag value.
+func (key ObjectKey) SealETag(etag []byte) []byte {
+	if len(etag) == 0 { // don't encrypt empty ETag - only if client sent ETag = MD5(object)
+		return etag
+	}
+	var buffer bytes.Buffer
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write([]byte("SSE-etag"))
+	if _, err := sio.Encrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to encrypt ETag using object key"))
+	}
+	return buffer.Bytes()
+}
+
+// UnsealETag unseals the etag using the provided object key.
+// It does not try to decrypt the ETag if len(etag) == 16
+// because such ETags indicate that the S3 client hasn't sent
+// an ETag = MD5(object) and the backend has picked an ETag value.
+func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) {
+	if !IsETagSealed(etag) {
+		return etag, nil
+	}
+	var buffer bytes.Buffer
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write([]byte("SSE-etag"))
+	if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}

cmd/crypto/key_test.go

@@ -0,0 +1,196 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"io"
+	"testing"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
+
+func recoverTest(i int, shouldPass bool, t *testing.T) {
+	if err := recover(); err == nil && !shouldPass {
+		t.Errorf("Test %d should fail but passed successfully", i)
+	} else if err != nil && shouldPass {
+		t.Errorf("Test %d should pass but failed: %v", i, err)
+	}
+}
+
+var generateKeyTests = []struct {
+	ExtKey     [32]byte
+	Random     io.Reader
+	ShouldPass bool
+}{
+	{ExtKey: [32]byte{}, Random: nil, ShouldPass: true},              // 0
+	{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true},      // 1
+	{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true},  // 2
+	{ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3
+}
+
+func TestGenerateKey(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+
+	for i, test := range generateKeyTests {
+		i, test := i, test
+		func() {
+			defer recoverTest(i, test.ShouldPass, t)
+			key := GenerateKey(test.ExtKey, test.Random)
+			if [32]byte(key) == [32]byte{} {
+				t.Errorf("Test %d: generated key is zero key", i) // check that we generate random and unique key
+			}
+		}()
+	}
+}
+
+var generateIVTests = []struct {
+	Random     io.Reader
+	ShouldPass bool
+}{
+	{Random: nil, ShouldPass: true},              // 0
+	{Random: rand.Reader, ShouldPass: true},      // 1
+	{Random: shortRandom(32), ShouldPass: true},  // 2
+	{Random: shortRandom(31), ShouldPass: false}, // 3
+}
+
+func TestGenerateIV(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+
+	for i, test := range generateIVTests {
+		i, test := i, test
+		func() {
+			defer recoverTest(i, test.ShouldPass, t)
+			iv := GenerateIV(test.Random)
+			if iv == [32]byte{} {
+				t.Errorf("Test %d: generated IV is zero IV", i) // check that we generate random and unique IV
+			}
+		}()
+	}
+}
+
+var sealUnsealKeyTests = []struct {
+	SealExtKey, SealIV                 [32]byte
+	SealDomain, SealBucket, SealObject string
+
+	UnsealExtKey                             [32]byte
+	UnsealDomain, UnsealBucket, UnsealObject string
+
+	ShouldPass bool
+}{
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: true,
+	}, // 0
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{1}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", // different ext-key
+		ShouldPass: false,
+	}, // 1
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-S3", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", // different domain
+		ShouldPass: false,
+	}, // 2
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "Bucket", UnsealObject: "object", // different bucket
+		ShouldPass: false,
+	}, // 3
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "Object", // different object
+		ShouldPass: false,
+	}, // 4
+}
+
+func TestSealUnsealKey(t *testing.T) {
+	for i, test := range sealUnsealKeyTests {
+		key := GenerateKey(test.SealExtKey, rand.Reader)
+		sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealDomain, test.SealBucket, test.SealObject)
+		if err := key.Unseal(test.UnsealExtKey, sealedKey, test.UnsealDomain, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
+			t.Errorf("Test %d should fail but passed successfully", i)
+		} else if err != nil && test.ShouldPass {
+			t.Errorf("Test %d should pass put failed: %v", i, err)
+		}
+	}
+
+	// Test legacy InsecureSealAlgorithm
+	var extKey, iv [32]byte
+	key := GenerateKey(extKey, rand.Reader)
+	sealedKey := key.Seal(extKey, iv, "SSE-S3", "bucket", "object")
+	sealedKey.Algorithm = InsecureSealAlgorithm
+	if err := key.Unseal(extKey, sealedKey, "SSE-S3", "bucket", "object"); err == nil {
+		t.Errorf("'%s' test succeeded but it should fail because the legacy algorithm was used", sealedKey.Algorithm)
+	}
+}
+
+var derivePartKeyTest = []struct {
+	PartID  uint32
+	PartKey string
+}{
+	{PartID: 0, PartKey: "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"},
+	{PartID: 1, PartKey: "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"},
+	{PartID: 10000, PartKey: "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"},
+}
+
+func TestDerivePartKey(t *testing.T) {
+	var key ObjectKey
+	for i, test := range derivePartKeyTest {
+		expectedPartKey, err := hex.DecodeString(test.PartKey)
+		if err != nil {
+			t.Fatalf("Test %d failed to decode expected part-key: %v", i, err)
+		}
+		partKey := key.DerivePartKey(test.PartID)
+		if !bytes.Equal(partKey[:], expectedPartKey[:]) {
+			t.Errorf("Test %d derives wrong part-key: got '%s' want: '%s'", i, hex.EncodeToString(partKey[:]), test.PartKey)
+		}
+	}
+}
+
+var sealUnsealETagTests = []string{
+	"",
+	"90682b8e8cc7609c",
+	"90682b8e8cc7609c4671e1d64c73fc30",
+	"90682b8e8cc7609c4671e1d64c73fc307fb3104f",
+}
+
+func TestSealETag(t *testing.T) {
+	var key ObjectKey
+	for i := range key {
+		key[i] = byte(i)
+	}
+	for i, etag := range sealUnsealETagTests {
+		tag, err := hex.DecodeString(etag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decode etag: %s", i, err)
+		}
+		sealedETag := key.SealETag(tag)
+		unsealedETag, err := key.UnsealETag(sealedETag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decrypt etag: %s", i, err)
+		}
+		if !bytes.Equal(unsealedETag, tag) {
+			t.Errorf("Test %d: unsealed etag does not match: got %s - want %s", i, hex.EncodeToString(unsealedETag), etag)
+		}
+	}
+}

cmd/crypto/kms.go

@@ -0,0 +1,138 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+// Context is a list of key-value pairs cryptographically
+// associated with a certain object.
+type Context map[string]string
+
+// WriteTo writes the context in a canonical from to w.
+// It returns the number of bytes and the first error
+// encounter during writing to w, if any.
+//
+// WriteTo sorts the context keys and writes the sorted
+// key-value pairs as canonical JSON object to w.
+func (c Context) WriteTo(w io.Writer) (n int64, err error) {
+	sortedKeys := make(sort.StringSlice, 0, len(c))
+	for k := range c {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Sort(sortedKeys)
+
+	nn, err := io.WriteString(w, "{")
+	if err != nil {
+		return n + int64(nn), err
+	}
+	n += int64(nn)
+	for i, k := range sortedKeys {
+		s := fmt.Sprintf("\"%s\":\"%s\",", k, c[k])
+		if i == len(sortedKeys)-1 {
+			s = s[:len(s)-1] // remove last ','
+		}
+
+		nn, err = io.WriteString(w, s)
+		if err != nil {
+			return n + int64(nn), err
+		}
+		n += int64(nn)
+	}
+	nn, err = io.WriteString(w, "}")
+	return n + int64(nn), err
+}
+
+// KMS represents an active and authenticted connection
+// to a Key-Management-Service. It supports generating
+// data key generation and unsealing of KMS-generated
+// data keys.
+type KMS interface {
+	// GenerateKey generates a new random data key using
+	// the master key referenced by the keyID. It returns
+	// the plaintext key and the sealed plaintext key
+	// on success.
+	//
+	// The context is cryptographically bound to the
+	// generated key. The same context must be provided
+	// again to unseal the generated key.
+	GenerateKey(keyID string, context Context) (key [32]byte, sealedKey []byte, err error)
+
+	// UnsealKey unseals the sealedKey using the master key
+	// referenced by the keyID. The provided context must
+	// match the context used to generate the sealed key.
+	UnsealKey(keyID string, sealedKey []byte, context Context) (key [32]byte, err error)
+}
+
+type masterKeyKMS struct {
+	masterKey [32]byte
+}
+
+// NewKMS returns a basic KMS implementation from a single 256 bit master key.
+//
+// The KMS accepts any keyID but binds the keyID and context cryptographically
+// to the generated keys.
+func NewKMS(key [32]byte) KMS { return &masterKeyKMS{masterKey: key} }
+
+func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
+	if _, err = io.ReadFull(rand.Reader, key[:]); err != nil {
+		logger.CriticalIf(context.Background(), errOutOfEntropy)
+	}
+
+	var (
+		buffer     bytes.Buffer
+		derivedKey = kms.deriveKey(keyID, ctx)
+	)
+	if n, err := sio.Encrypt(&buffer, bytes.NewReader(key[:]), sio.Config{Key: derivedKey[:]}); err != nil || n != 64 {
+		logger.CriticalIf(context.Background(), errors.New("KMS: unable to encrypt data key"))
+	}
+	sealedKey = buffer.Bytes()
+	return key, sealedKey, nil
+}
+
+func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
+	var (
+		buffer     bytes.Buffer
+		derivedKey = kms.deriveKey(keyID, ctx)
+	)
+	if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 {
+		return key, err // TODO(aead): upgrade sio to use sio.Error
+	}
+	copy(key[:], buffer.Bytes())
+	return key, nil
+}
+
+func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte) {
+	if context == nil {
+		context = Context{}
+	}
+	mac := hmac.New(sha256.New, kms.masterKey[:])
+	mac.Write([]byte(keyID))
+	context.WriteTo(mac)
+	mac.Sum(key[:0])
+	return key
+}

cmd/crypto/kms_test.go

@@ -0,0 +1,84 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"path"
+	"strings"
+	"testing"
+)
+
+var masterKeyKMSTests = []struct {
+	GenKeyID, UnsealKeyID     string
+	GenContext, UnsealContext Context
+
+	ShouldFail bool
+}{
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{}, UnsealContext: nil, ShouldFail: false},                                                                                     // 0
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{}, UnsealContext: Context{}, ShouldFail: false},                                                               // 1
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{"bucket": "object"}, UnsealContext: Context{"bucket": "object"}, ShouldFail: false},                           // 2
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"bucket": path.Join("bucket", "object")}, UnsealContext: Context{"bucket": path.Join("bucket", "object")}, ShouldFail: false}, // 3
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"a": "a", "0": "0", "b": "b"}, UnsealContext: Context{"b": "b", "a": "a", "0": "0"}, ShouldFail: false},                       // 4
+
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7e", GenContext: Context{}, UnsealContext: Context{}, ShouldFail: true},                                                               // 5
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{"bucket": "object"}, UnsealContext: Context{"Bucket": "object"}, ShouldFail: true},                           // 6
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"bucket": path.Join("bucket", "Object")}, UnsealContext: Context{"bucket": path.Join("bucket", "object")}, ShouldFail: true}, // 7
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"a": "a", "0": "1", "b": "b"}, UnsealContext: Context{"b": "b", "a": "a", "0": "0"}, ShouldFail: true},                       // 8
+}
+
+func TestMasterKeyKMS(t *testing.T) {
+	kms := NewKMS([32]byte{})
+	for i, test := range masterKeyKMSTests {
+		key, sealedKey, err := kms.GenerateKey(test.GenKeyID, test.GenContext)
+		if err != nil {
+			t.Errorf("Test %d: KMS failed to generate key: %v", i, err)
+		}
+		unsealedKey, err := kms.UnsealKey(test.UnsealKeyID, sealedKey, test.UnsealContext)
+		if err != nil && !test.ShouldFail {
+			t.Errorf("Test %d: KMS failed to unseal the generated key: %v", i, err)
+		}
+		if err == nil && test.ShouldFail {
+			t.Errorf("Test %d: KMS unsealed the generated successfully but should have failed", i)
+		}
+		if !test.ShouldFail && !bytes.Equal(key[:], unsealedKey[:]) {
+			t.Errorf("Test %d: The generated and unsealed key differ", i)
+		}
+	}
+}
+
+var contextWriteToTests = []struct {
+	Context      Context
+	ExpectedJSON string
+}{
+	{Context: Context{}, ExpectedJSON: "{}"},                                                    // 0
+	{Context: Context{"a": "b"}, ExpectedJSON: `{"a":"b"}`},                                     // 1
+	{Context: Context{"a": "b", "c": "d"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 2
+	{Context: Context{"c": "d", "a": "b"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 3
+	{Context: Context{"0": "1", "-": "2", ".": "#"}, ExpectedJSON: `{"-":"2",".":"#","0":"1"}`}, // 4
+}
+
+func TestContextWriteTo(t *testing.T) {
+	for i, test := range contextWriteToTests {
+		var jsonContext strings.Builder
+		if _, err := test.Context.WriteTo(&jsonContext); err != nil {
+			t.Errorf("Test %d: Failed to encode context: %v", i, err)
+			continue
+		}
+		if s := jsonContext.String(); s != test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON)
+		}
+	}
+}

cmd/crypto/metadata.go

@@ -0,0 +1,236 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// IsMultiPart returns true if the object metadata indicates
+// that it was uploaded using some form of server-side-encryption
+// and the S3 multipart API.
+func IsMultiPart(metadata map[string]string) bool {
+	if _, ok := metadata[SSEMultipart]; ok {
+		return true
+	}
+	return false
+}
+
+// RemoveSensitiveEntries removes confidential encryption
+// information - e.g. the SSE-C key - from the metadata map.
+// It has the same semantics as RemoveSensitiveHeaders.
+func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
+	delete(metadata, SSECKey)
+	delete(metadata, SSECopyKey)
+}
+
+// RemoveInternalEntries removes all crypto-specific internal
+// metadata entries from the metadata map.
+func RemoveInternalEntries(metadata map[string]string) {
+	delete(metadata, SSEMultipart)
+	delete(metadata, SSEIV)
+	delete(metadata, SSESealAlgorithm)
+	delete(metadata, SSECSealedKey)
+	delete(metadata, S3SealedKey)
+	delete(metadata, S3KMSKeyID)
+	delete(metadata, S3KMSSealedKey)
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that it was uploaded using some form of server-side-encryption.
+//
+// IsEncrypted only checks whether the metadata contains at least
+// one entry indicating SSE-C or SSE-S3.
+func IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[SSEIV]; ok {
+		return true
+	}
+	if _, ok := metadata[SSESealAlgorithm]; ok {
+		return true
+	}
+	if IsMultiPart(metadata) {
+		return true
+	}
+	if S3.IsEncrypted(metadata) {
+		return true
+	}
+	if SSEC.IsEncrypted(metadata) {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-S3.
+func (s3) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[S3SealedKey]; ok {
+		return true
+	}
+	if _, ok := metadata[S3KMSKeyID]; ok {
+		return true
+	}
+	if _, ok := metadata[S3KMSSealedKey]; ok {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-C.
+func (ssec) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[SSECSealedKey]; ok {
+		return true
+	}
+	return false
+}
+
+// CreateMultipartMetadata adds the multipart flag entry to metadata
+// and returns modifed metadata. It allocates a new metadata map if
+// metadata is nil.
+func CreateMultipartMetadata(metadata map[string]string) map[string]string {
+	if metadata == nil {
+		metadata = map[string]string{}
+	}
+	metadata[SSEMultipart] = ""
+	return metadata
+}
+
+// CreateMetadata encodes the keyID, the sealed kms data key and the sealed key
+// into the metadata and returns the modified metadata. It allocates a new
+// metadata map if metadata is nil.
+func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+	}
+
+	if metadata == nil {
+		metadata = map[string]string{}
+	}
+	metadata[S3KMSKeyID] = keyID
+	metadata[SSESealAlgorithm] = sealedKey.Algorithm
+	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[S3SealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-S3 related values from the object metadata
+// and checks whether they are well-formed. It returns the KMS key-ID, the
+// sealed KMS key and the sealed object key on success.
+func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[SSEIV]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[SSESealAlgorithm]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[S3SealedKey]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal sealed key for SSE-S3"}
+	}
+	keyID, ok = metadata[S3KMSKeyID]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal KMS key-ID for SSE-S3"}
+	}
+	b64KMSSealedKey, ok := metadata[S3KMSSealedKey]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal sealed KMS data key for SSE-S3"}
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm {
+		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return keyID, kmsKey, sealedKey, Error{"The internal sealed key for SSE-S3 is invalid"}
+	}
+	kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+	if err != nil {
+		return keyID, kmsKey, sealedKey, Error{"The internal sealed KMS data key for SSE-S3 is invalid"}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return keyID, kmsKey, sealedKey, nil
+}
+
+// CreateMetadata encodes the sealed key into the metadata and returns the modified metadata.
+// It allocates a new metadata map if metadata is nil.
+func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
+	}
+
+	if metadata == nil {
+		metadata = map[string]string{}
+	}
+	metadata[SSESealAlgorithm] = SealAlgorithm
+	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[SSECSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-C related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success.
+func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[SSEIV]
+	if !ok {
+		return sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[SSESealAlgorithm]
+	if !ok {
+		return sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[SSECSealedKey]
+	if !ok {
+		return sealedKey, Error{"The object metadata is missing the internal sealed key for SSE-C"}
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
+		return sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return sealedKey, Error{"The internal sealed key for SSE-C is invalid"}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return sealedKey, nil
+}
+
+// IsETagSealed returns true if the etag seems to be encrypted.
+func IsETagSealed(etag []byte) bool { return len(etag) > 16 }

cmd/crypto/metadata_test.go

@@ -0,0 +1,439 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/hex"
+	"testing"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+var isMultipartTests = []struct {
+	Metadata  map[string]string
+	Multipart bool
+}{
+	{Multipart: true, Metadata: map[string]string{SSEMultipart: ""}},                           // 0
+	{Multipart: true, Metadata: map[string]string{"X-Minio-Internal-Encrypted-Multipart": ""}}, // 1
+	{Multipart: true, Metadata: map[string]string{SSEMultipart: "some-value"}},                 // 2
+	{Multipart: false, Metadata: map[string]string{"": ""}},                                    // 3
+	{Multipart: false, Metadata: map[string]string{"X-Minio-Internal-EncryptedMultipart": ""}}, // 4
+}
+
+func TestIsMultipart(t *testing.T) {
+	for i, test := range isMultipartTests {
+		if isMultipart := IsMultiPart(test.Metadata); isMultipart != test.Multipart {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isMultipart, test.Multipart)
+		}
+	}
+}
+
+var isEncryptedTests = []struct {
+	Metadata  map[string]string
+	Encrypted bool
+}{
+	{Encrypted: true, Metadata: map[string]string{SSEMultipart: ""}},                               // 0
+	{Encrypted: true, Metadata: map[string]string{SSEIV: ""}},                                      // 1
+	{Encrypted: true, Metadata: map[string]string{SSESealAlgorithm: ""}},                           // 2
+	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
+	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
+	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
+	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
+	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
+}
+
+func TestIsEncrypted(t *testing.T) {
+	for i, test := range isEncryptedTests {
+		if isEncrypted := IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
+		}
+	}
+}
+
+var s3IsEncryptedTests = []struct {
+	Metadata  map[string]string
+	Encrypted bool
+}{
+	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
+	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
+	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
+	{Encrypted: false, Metadata: map[string]string{SSECSealedKey: ""}},                             // 3
+	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
+	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
+	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
+	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
+}
+
+func TestS3IsEncrypted(t *testing.T) {
+	for i, test := range s3IsEncryptedTests {
+		if isEncrypted := S3.IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
+		}
+	}
+}
+
+var ssecIsEncryptedTests = []struct {
+	Metadata  map[string]string
+	Encrypted bool
+}{
+	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
+	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
+	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
+	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
+	{Encrypted: false, Metadata: map[string]string{S3SealedKey: ""}},                               // 4
+	{Encrypted: false, Metadata: map[string]string{S3KMSKeyID: ""}},                                // 5
+	{Encrypted: false, Metadata: map[string]string{S3KMSSealedKey: ""}},                            // 6
+	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
+	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
+}
+
+func TestSSECIsEncrypted(t *testing.T) {
+	for i, test := range ssecIsEncryptedTests {
+		if isEncrypted := SSEC.IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
+		}
+	}
+}
+
+var s3ParseMetadataTests = []struct {
+	Metadata    map[string]string
+	ExpectedErr error
+
+	DataKey   []byte
+	KeyID     string
+	SealedKey SealedKey
+}{
+	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}}, // 0
+	{
+		ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 1
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal sealed key for SSE-S3"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 2
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal KMS key-ID for SSE-S3"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 3
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal sealed KMS data key for SSE-S3"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
+		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 4
+	{
+		ExpectedErr: errInvalidInternalIV,
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: ""},
+		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 5
+	{
+		ExpectedErr: errInvalidInternalSealAlgorithm,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: "",
+		},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 6
+	{
+		ExpectedErr: Error{"The internal sealed key for SSE-S3 is invalid"},
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
+			S3KMSKeyID: "", S3KMSSealedKey: "",
+		},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 7
+	{
+		ExpectedErr: Error{"The internal sealed KMS data key for SSE-S3 is invalid"},
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
+			S3KMSSealedKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
+		},
+		DataKey: []byte{}, KeyID: "key-1", SealedKey: SealedKey{},
+	}, // 8
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "", S3KMSSealedKey: "",
+		},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{Algorithm: SealAlgorithm},
+	}, // 9
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: SealAlgorithm,
+			S3SealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), S3KMSKeyID: "key-1",
+			S3KMSSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
+		},
+		DataKey: make([]byte, 48), KeyID: "key-1", SealedKey: SealedKey{Algorithm: SealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
+	}, // 10
+}
+
+func TestS3ParseMetadata(t *testing.T) {
+	for i, test := range s3ParseMetadataTests {
+		keyID, dataKey, sealedKey, err := S3.ParseMetadata(test.Metadata)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if !bytes.Equal(dataKey, test.DataKey) {
+			t.Errorf("Test %d: got data key '%v' - want data key '%v'", i, dataKey, test.DataKey)
+		}
+		if keyID != test.KeyID {
+			t.Errorf("Test %d: got key-ID '%v' - want key-ID '%v'", i, keyID, test.KeyID)
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: got sealed key '%v' - want sealed key '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: got sealed key IV '%v' - want sealed key IV '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+	}
+}
+
+var ssecParseMetadataTests = []struct {
+	Metadata    map[string]string
+	ExpectedErr error
+
+	SealedKey SealedKey
+}{
+	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
+	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal sealed key for SSE-C"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
+	}, // 2
+	{
+		ExpectedErr: errInvalidInternalIV,
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", SSECSealedKey: ""}, SealedKey: SealedKey{},
+	}, // 3
+	{
+		ExpectedErr: errInvalidInternalSealAlgorithm,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", SSECSealedKey: "",
+		},
+		SealedKey: SealedKey{},
+	}, // 4
+	{
+		ExpectedErr: Error{"The internal sealed key for SSE-C is invalid"},
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
+		},
+		SealedKey: SealedKey{},
+	}, // 5
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			SSECSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+		},
+		SealedKey: SealedKey{Algorithm: SealAlgorithm},
+	}, // 6
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: InsecureSealAlgorithm,
+			SSECSealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
+		},
+		SealedKey: SealedKey{Algorithm: InsecureSealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
+	}, // 7
+}
+
+func TestCreateMultipartMetadata(t *testing.T) {
+	metadata := CreateMultipartMetadata(nil)
+	if v, ok := metadata[SSEMultipart]; !ok || v != "" {
+		t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", SSEMultipart, v, "")
+	}
+}
+
+func TestSSECParseMetadata(t *testing.T) {
+	for i, test := range ssecParseMetadataTests {
+		sealedKey, err := SSEC.ParseMetadata(test.Metadata)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: got sealed key '%v' - want sealed key '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: got sealed key IV '%v' - want sealed key IV '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+	}
+}
+
+var s3CreateMetadataTests = []struct {
+	KeyID         string
+	SealedDataKey []byte
+	SealedKey     SealedKey
+}{
+	{KeyID: "", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}},
+}
+
+func TestS3CreateMetadata(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+	for i, test := range s3CreateMetadataTests {
+		metadata := S3.CreateMetadata(nil, test.KeyID, test.SealedDataKey, test.SealedKey)
+		keyID, kmsKey, sealedKey, err := S3.ParseMetadata(metadata)
+		if err != nil {
+			t.Errorf("Test %d: failed to parse metadata: %v", i, err)
+			continue
+		}
+		if keyID != test.KeyID {
+			t.Errorf("Test %d: Key-ID mismatch: got '%s' - want '%s'", i, keyID, test.KeyID)
+		}
+		if !bytes.Equal(kmsKey, test.SealedDataKey) {
+			t.Errorf("Test %d: sealed KMS data mismatch: got '%v' - want '%v'", i, kmsKey, test.SealedDataKey)
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: seal algorithm mismatch: got '%s' - want '%s'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: IV mismatch: got '%v' - want '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: sealed key mismatch: got '%v' - want '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+	}
+
+	defer func() {
+		if err := recover(); err == nil || err != logger.ErrCritical {
+			t.Errorf("Expected '%s' panic for invalid seal algorithm but got '%s'", logger.ErrCritical, err)
+		}
+	}()
+	_ = S3.CreateMetadata(nil, "", []byte{}, SealedKey{Algorithm: InsecureSealAlgorithm})
+}
+
+var ssecCreateMetadataTests = []struct {
+	KeyID         string
+	SealedDataKey []byte
+	SealedKey     SealedKey
+}{
+	{KeyID: "", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}},
+}
+
+func TestSSECCreateMetadata(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+	for i, test := range ssecCreateMetadataTests {
+		metadata := SSEC.CreateMetadata(nil, test.SealedKey)
+		sealedKey, err := SSEC.ParseMetadata(metadata)
+		if err != nil {
+			t.Errorf("Test %d: failed to parse metadata: %v", i, err)
+			continue
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: seal algorithm mismatch: got '%s' - want '%s'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: IV mismatch: got '%v' - want '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: sealed key mismatch: got '%v' - want '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+	}
+
+	defer func() {
+		if err := recover(); err == nil || err != logger.ErrCritical {
+			t.Errorf("Expected '%s' panic for invalid seal algorithm but got '%s'", logger.ErrCritical, err)
+		}
+	}()
+	_ = SSEC.CreateMetadata(nil, SealedKey{Algorithm: InsecureSealAlgorithm})
+}
+
+var isETagSealedTests = []struct {
+	ETag     string
+	IsSealed bool
+}{
+	{ETag: "", IsSealed: false},                                                                                                // 0
+	{ETag: "90682b8e8cc7609c4671e1d64c73fc30", IsSealed: false},                                                                // 1
+	{ETag: "f201040c9dc593e39ea004dc1323699bcd", IsSealed: true},                                                               // 2 not valid ciphertext but looks like sealed ETag
+	{ETag: "20000f00fba2ee2ae4845f725964eeb9e092edfabc7ab9f9239e8344341f769a51ce99b4801b0699b92b16a72fa94972", IsSealed: true}, // 3
+}
+
+func TestIsETagSealed(t *testing.T) {
+	for i, test := range isETagSealedTests {
+		etag, err := hex.DecodeString(test.ETag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decode etag: %s", i, err)
+		}
+		if sealed := IsETagSealed(etag); sealed != test.IsSealed {
+			t.Errorf("Test %d: got %v - want %v", i, sealed, test.IsSealed)
+		}
+	}
+}
+
+var removeInternalEntriesTests = []struct {
+	Metadata, Expected map[string]string
+}{
+	{ // 0
+		Metadata: map[string]string{
+			SSEMultipart:     "",
+			SSEIV:            "",
+			SSESealAlgorithm: "",
+			SSECSealedKey:    "",
+			S3SealedKey:      "",
+			S3KMSKeyID:       "",
+			S3KMSSealedKey:   "",
+		},
+		Expected: map[string]string{},
+	},
+	{ // 1
+		Metadata: map[string]string{
+			SSEMultipart:         "",
+			SSEIV:                "",
+			"X-Amz-Meta-A":       "X",
+			"X-Minio-Internal-B": "Y",
+		},
+		Expected: map[string]string{
+			"X-Amz-Meta-A":       "X",
+			"X-Minio-Internal-B": "Y",
+		},
+	},
+}
+
+func TestRemoveInternalEntries(t *testing.T) {
+	isEqual := func(x, y map[string]string) bool {
+		if len(x) != len(y) {
+			return false
+		}
+		for k, v := range x {
+			if u, ok := y[k]; !ok || v != u {
+				return false
+			}
+		}
+		return true
+	}
+
+	for i, test := range removeInternalEntriesTests {
+		RemoveInternalEntries(test.Metadata)
+		if !isEqual(test.Metadata, test.Expected) {
+			t.Errorf("Test %d: got %v - want %v", i, test.Metadata, test.Expected)
+		}
+	}
+}

cmd/crypto/sse.go

@@ -0,0 +1,157 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"path"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/ioutil"
+	"github.com/minio/sio"
+)
+
+const (
+	// SSEMultipart is the metadata key indicating that the object
+	// was uploaded using the S3 multipart API and stored using
+	// some from of server-side-encryption.
+	SSEMultipart = "X-Minio-Internal-Encrypted-Multipart"
+
+	// SSEIV is the metadata key referencing the random initialization
+	// vector (IV) used for SSE-S3 and SSE-C key derivation.
+	SSEIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
+
+	// SSESealAlgorithm is the metadata key referencing the algorithm
+	// used by SSE-C and SSE-S3 to encrypt the object.
+	SSESealAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
+
+	// SSECSealedKey is the metadata key referencing the sealed object-key for SSE-C.
+	SSECSealedKey = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
+
+	// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
+	S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
+
+	// S3KMSKeyID is the metadata key referencing the KMS key-id used to
+	// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
+	S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
+
+	// S3KMSSealedKey is the metadata key referencing the encrypted key generated
+	// by KMS. It is only used for SSE-S3 + KMS.
+	S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
+)
+
+const (
+	// SealAlgorithm is the encryption/sealing algorithm used to derive & seal
+	// the key-encryption-key and to en/decrypt the object data.
+	SealAlgorithm = "DAREv2-HMAC-SHA256"
+
+	// InsecureSealAlgorithm is the legacy encryption/sealing algorithm used
+	// to derive & seal the key-encryption-key and to en/decrypt the object data.
+	// This algorithm should not be used for new objects because its key derivation
+	// is not optimal. See: https://github.com/minio/minio/pull/6121
+	InsecureSealAlgorithm = "DARE-SHA256"
+)
+
+// String returns the SSE domain as string. For SSE-S3 the
+// domain is "SSE-S3".
+func (s3) String() string { return "SSE-S3" }
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using KMS and returns the decrypted object
+// key.
+func (sse s3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	keyID, kmsKey, sealedKey, err := sse.ParseMetadata(metadata)
+	if err != nil {
+		return
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return
+	}
+	err = key.Unseal(unsealKey, sealedKey, sse.String(), bucket, object)
+	return
+}
+
+// String returns the SSE domain as string. For SSE-C the
+// domain is "SSE-C".
+func (ssec) String() string { return "SSE-C" }
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-C client key of the HTTP headers
+// and returns the decrypted object key.
+func (sse ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := sse.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-Copy client key of the HTTP headers
+// and returns the decrypted object key.
+func (sse ssecCopy) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := sse.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// unsealObjectKey decrypts and returns the sealed object key
+// from the metadata using the SSE-C client key.
+func unsealObjectKey(clientKey [32]byte, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	sealedKey, err := SSEC.ParseMetadata(metadata)
+	if err != nil {
+		return
+	}
+	err = key.Unseal(clientKey, sealedKey, SSEC.String(), bucket, object)
+	return
+}
+
+// EncryptSinglePart encrypts an io.Reader which must be the
+// the body of a single-part PUT request.
+func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
+	r, err := sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, Key: key[:]})
+	if err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to encrypt io.Reader using object key"))
+	}
+	return r
+}
+
+// EncryptMultiPart encrypts an io.Reader which must be the body of
+// multi-part PUT request. It derives an unique encryption key from
+// the partID and the object key.
+func EncryptMultiPart(r io.Reader, partID int, key ObjectKey) io.Reader {
+	partKey := key.DerivePartKey(uint32(partID))
+	return EncryptSinglePart(r, ObjectKey(partKey))
+}
+
+// DecryptSinglePart decrypts an io.Writer which must an object
+// uploaded with the single-part PUT API. The offset and length
+// specify the requested range.
+func DecryptSinglePart(w io.Writer, offset, length int64, key ObjectKey) io.WriteCloser {
+	const PayloadSize = 1 << 16 // DARE 2.0
+	w = ioutil.LimitedWriter(w, offset%PayloadSize, length)
+
+	decWriter, err := sio.DecryptWriter(w, sio.Config{Key: key[:]})
+	if err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to decrypt io.Writer using object key"))
+	}
+	return decWriter
+}

cmd/crypto/sse_test.go

@@ -0,0 +1,226 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"net/http"
+	"testing"
+)
+
+func TestS3String(t *testing.T) {
+	const Domain = "SSE-S3"
+	if domain := S3.String(); domain != Domain {
+		t.Errorf("S3's string method returns wrong domain: got '%s' - want '%s'", domain, Domain)
+	}
+}
+
+func TestSSECString(t *testing.T) {
+	const Domain = "SSE-C"
+	if domain := SSEC.String(); domain != Domain {
+		t.Errorf("SSEC's string method returns wrong domain: got '%s' - want '%s'", domain, Domain)
+	}
+}
+
+var ssecUnsealObjectKeyTests = []struct {
+	Headers        http.Header
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid HTTP headers and valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object2",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrSecretKeyMismatch,
+	},
+	{ // 2 - Valid HTTP headers but invalid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":         "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+	{ // 3 - Invalid HTTP headers for valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5,
+	},
+}
+
+func TestSSECUnsealObjectKey(t *testing.T) {
+	for i, test := range ssecUnsealObjectKeyTests {
+		if _, err := SSEC.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}
+
+var sseCopyUnsealObjectKeyTests = []struct {
+	Headers        http.Header
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid HTTP headers and valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object2",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrSecretKeyMismatch,
+	},
+	{ // 2 - Valid HTTP headers but invalid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":         "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+	{ // 3 - Invalid HTTP headers for valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5,
+	},
+}
+
+func TestSSECopyUnsealObjectKey(t *testing.T) {
+	for i, test := range sseCopyUnsealObjectKeyTests {
+		if _, err := SSECopy.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}
+
+var s3UnsealObjectKeyTests = []struct {
+	KMS            KMS
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid KMS key-ID and valid metadata entries for bucket/object
+		KMS:    NewKMS([32]byte{}),
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Iv":                "hhVY0LKR1YtZbzAKxTWUfZt5enDfYX6Fxz1ma8Kiudc=",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key":     "IAAfALhsOeD5AE3s5Zgq3DZ5VFGsOa3B0ksVC86veDcaj+fXv2U0VadhPaOKYr9Emd5ssOsO0uIhIIrKiOy9rA==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAMRS2iw45FsfiF3QXajSYVWj1lxMpQm6DxDGPtADCX6fJQQ4atHBtfpgqJFyeQmIHsm0FBI+UlHw1Lv4ug==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id":     "test-key-1",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm":    "DAREv2-HMAC-SHA256",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid KMS key-ID for invalid metadata entries for bucket/object
+		KMS:    NewKMS([32]byte{}),
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Iv":                "hhVY0LKR1YtZbzAKxTWUfZt5enDfYX6Fxz1ma8Kiudc=",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key":     "IAAfALhsOeD5AE3s5Zgq3DZ5VFGsOa3B0ksVC86veDcaj+fXv2U0VadhPaOKYr9Emd5ssOsO0uIhIIrKiOy9rA==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAMRS2iw45FsfiF3QXajSYVWj1lxMpQm6DxDGPtADCX6fJQQ4atHBtfpgqJFyeQmIHsm0FBI+UlHw1Lv4ug==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id":     "test-key-1",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+}
+
+func TestS3UnsealObjectKey(t *testing.T) {
+	for i, test := range s3UnsealObjectKeyTests {
+		if _, err := S3.UnsealObjectKey(test.KMS, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}

cmd/crypto/vault.go

@@ -0,0 +1,270 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	vault "github.com/hashicorp/vault/api"
+)
+
+const (
+	// vaultEndpointEnv Vault endpoint environment variable
+	vaultEndpointEnv = "MINIO_SSE_VAULT_ENDPOINT"
+	// vaultAuthTypeEnv type of vault auth to be used
+	vaultAuthTypeEnv = "MINIO_SSE_VAULT_AUTH_TYPE"
+	// vaultAppRoleIDEnv  Vault AppRole ID environment variable
+	vaultAppRoleIDEnv = "MINIO_SSE_VAULT_APPROLE_ID"
+	// vaultAppSecretIDEnv Vault AppRole Secret environment variable
+	vaultAppSecretIDEnv = "MINIO_SSE_VAULT_APPROLE_SECRET"
+	// vaultKeyVersionEnv Vault Key Version environment variable
+	vaultKeyVersionEnv = "MINIO_SSE_VAULT_KEY_VERSION"
+	// vaultKeyNameEnv Vault Encryption Key Name environment variable
+	vaultKeyNameEnv = "MINIO_SSE_VAULT_KEY_NAME"
+
+	// vaultCAPath is the path to a directory of PEM-encoded CA
+	// cert files to verify the Vault server SSL certificate.
+	vaultCAPath = "MINIO_SSE_VAULT_CAPATH"
+)
+
+var (
+	//ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
+	ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
+)
+
+type vaultService struct {
+	config        *VaultConfig
+	client        *vault.Client
+	leaseDuration time.Duration
+}
+
+// return transit secret engine's path for generate data key operation
+func (v *vaultService) genDataKeyEndpoint(key string) string {
+	return "/transit/datakey/plaintext/" + key
+}
+
+// return transit secret engine's path for decrypt operation
+func (v *vaultService) decryptEndpoint(key string) string {
+	return "/transit/decrypt/" + key
+}
+
+// VaultKey represents vault encryption key-id name & version
+type VaultKey struct {
+	Name    string `json:"name"`
+	Version int    `json:"version"`
+}
+
+// VaultAuth represents vault auth type to use. For now, AppRole is the only supported
+// auth type.
+type VaultAuth struct {
+	Type    string       `json:"type"`
+	AppRole VaultAppRole `json:"approle"`
+}
+
+// VaultAppRole represents vault approle credentials
+type VaultAppRole struct {
+	ID     string `json:"id"`
+	Secret string `json:"secret"`
+}
+
+// VaultConfig holds config required to start vault service
+type VaultConfig struct {
+	Endpoint string    `json:"endpoint"`
+	Auth     VaultAuth `json:"auth"`
+	Key      VaultKey  `json:"key-id"`
+}
+
+// validate whether all required env variables needed to start vault service have
+// been set
+func validateVaultConfig(c *VaultConfig) error {
+	if c.Endpoint == "" {
+		return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", vaultEndpointEnv)
+	}
+	if strings.ToLower(c.Auth.Type) != "approle" {
+		return fmt.Errorf("Unsupported hashicorp vault auth type - %s", vaultAuthTypeEnv)
+	}
+	if c.Auth.AppRole.ID == "" {
+		return fmt.Errorf("Missing hashicorp vault AppRole ID - %s is empty", vaultAppRoleIDEnv)
+	}
+	if c.Auth.AppRole.Secret == "" {
+		return fmt.Errorf("Missing hashicorp vault AppSecret ID - %s is empty", vaultAppSecretIDEnv)
+	}
+	if c.Key.Name == "" {
+		return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyNameEnv)
+	}
+	if c.Key.Version < 0 {
+		return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyVersionEnv)
+	}
+	return nil
+}
+
+// authenticate to vault with app role id and app role secret, and get a client access token, lease duration
+func getVaultAccessToken(client *vault.Client, appRoleID, appSecret string) (token string, duration int, err error) {
+	data := map[string]interface{}{
+		"role_id":   appRoleID,
+		"secret_id": appSecret,
+	}
+	resp, e := client.Logical().Write("auth/approle/login", data)
+	if e != nil {
+		return token, duration, e
+	}
+	if resp.Auth == nil {
+		return token, duration, ErrKMSAuthLogin
+	}
+	return resp.Auth.ClientToken, resp.Auth.LeaseDuration, nil
+}
+
+// NewVaultConfig sets KMSConfig from environment
+// variables and performs validations.
+func NewVaultConfig() (KMSConfig, error) {
+	kc := KMSConfig{}
+	endpoint := os.Getenv(vaultEndpointEnv)
+	roleID := os.Getenv(vaultAppRoleIDEnv)
+	roleSecret := os.Getenv(vaultAppSecretIDEnv)
+	keyName := os.Getenv(vaultKeyNameEnv)
+	keyVersion := 0
+	authType := "approle"
+	if versionStr := os.Getenv(vaultKeyVersionEnv); versionStr != "" {
+		version, err := strconv.Atoi(versionStr)
+		if err != nil {
+			return kc, fmt.Errorf("Unable to parse %s value (`%s`)", vaultKeyVersionEnv, versionStr)
+		}
+		keyVersion = version
+	}
+	// return if none of the vault env variables are configured
+	if (endpoint == "") && (roleID == "") && (roleSecret == "") && (keyName == "") && (keyVersion == 0) {
+		return kc, nil
+	}
+	c := VaultConfig{
+		Endpoint: endpoint,
+		Auth: VaultAuth{
+			Type: authType,
+			AppRole: VaultAppRole{
+				ID:     roleID,
+				Secret: roleSecret,
+			},
+		},
+		Key: VaultKey{
+			Version: keyVersion,
+			Name:    keyName,
+		},
+	}
+	if err := validateVaultConfig(&c); err != nil {
+		return kc, err
+	}
+	kc.Vault = c
+	return kc, nil
+}
+
+// NewVault initializes Hashicorp Vault KMS by
+// authenticating to Vault with the credentials in KMSConfig,
+// and gets a client token for future api calls.
+func NewVault(kmsConf KMSConfig) (KMS, error) {
+	config := kmsConf.Vault
+	vconfig := &vault.Config{
+		Address: config.Endpoint,
+	}
+	if err := vconfig.ConfigureTLS(&vault.TLSConfig{
+		CAPath: os.Getenv(vaultCAPath),
+	}); err != nil {
+		return nil, err
+	}
+	c, err := vault.NewClient(vconfig)
+	if err != nil {
+		return nil, err
+	}
+
+	if ns, ok := os.LookupEnv("VAULT_NAMESPACE"); ok {
+		c.SetNamespace(ns)
+	}
+
+	accessToken, leaseDuration, err := getVaultAccessToken(c, config.Auth.AppRole.ID, config.Auth.AppRole.Secret)
+	if err != nil {
+		return nil, err
+	}
+	// authenticate and get the access token
+	c.SetToken(accessToken)
+	v := vaultService{client: c, config: &config, leaseDuration: time.Duration(leaseDuration)}
+	v.renewToken(c)
+	return &v, nil
+}
+
+func (v *vaultService) renewToken(c *vault.Client) {
+	retryDelay := 1 * time.Minute
+	go func() {
+		for {
+			s, err := c.Auth().Token().RenewSelf(int(v.leaseDuration))
+			if err != nil {
+				time.Sleep(retryDelay)
+				continue
+			}
+			nextRenew := s.Auth.LeaseDuration / 2
+			time.Sleep(time.Duration(nextRenew) * time.Second)
+		}
+	}()
+}
+
+// Generates a random plain text key, sealed plain text key from
+// Vault. It returns the plaintext key and sealed plaintext key on success
+func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
+	contextStream := new(bytes.Buffer)
+	ctx.WriteTo(contextStream)
+
+	payload := map[string]interface{}{
+		"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+	}
+	s, err1 := v.client.Logical().Write(v.genDataKeyEndpoint(keyID), payload)
+
+	if err1 != nil {
+		return key, sealedKey, err1
+	}
+	sealKey := s.Data["ciphertext"].(string)
+	plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
+	if err != nil {
+		return key, sealedKey, err1
+	}
+	copy(key[:], []byte(plainKey))
+	return key, []byte(sealKey), nil
+}
+
+// unsealKMSKey unseals the sealedKey using the Vault master key
+// referenced by the keyID. The plain text key is returned on success.
+func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
+	contextStream := new(bytes.Buffer)
+	ctx.WriteTo(contextStream)
+	payload := map[string]interface{}{
+		"ciphertext": string(sealedKey),
+		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+	}
+	s, err1 := v.client.Logical().Write(v.decryptEndpoint(keyID), payload)
+	if err1 != nil {
+		return key, err1
+	}
+	base64Key := s.Data["plaintext"].(string)
+	plainKey, err1 := base64.StdEncoding.DecodeString(base64Key)
+	if err1 != nil {
+		return key, err1
+	}
+	copy(key[:], []byte(plainKey))
+
+	return key, nil
+}

cmd/disk-cache-config.go

@@ -18,13 +18,18 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"path/filepath"
+	"strings"
+
+	"github.com/minio/minio/pkg/ellipses"
 )
 
 // CacheConfig represents cache config settings
 type CacheConfig struct {
 	Drives  []string `json:"drives"`
 	Expiry  int      `json:"expiry"`
+	MaxUse  int      `json:"maxuse"`
 	Exclude []string `json:"exclude"`
 }
 
@@ -40,6 +45,15 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
 	if err = json.Unmarshal(data, _cfg); err != nil {
 		return err
 	}
+
+	if _cfg.Expiry < 0 {
+		return errors.New("config expiry value should not be negative")
+	}
+
+	if _cfg.MaxUse < 0 {
+		return errors.New("config max use value should not be null or negative")
+	}
+
 	if _, err = parseCacheDrives(_cfg.Drives); err != nil {
 		return err
 	}
@@ -51,12 +65,42 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
 
 // Parses given cacheDrivesEnv and returns a list of cache drives.
 func parseCacheDrives(drives []string) ([]string, error) {
+	if len(drives) == 0 {
+		return drives, nil
+	}
+	var endpoints []string
 	for _, d := range drives {
+		if ellipses.HasEllipses(d) {
+			s, err := parseCacheDrivePaths(d)
+			if err != nil {
+				return nil, err
+			}
+			endpoints = append(endpoints, s...)
+		} else {
+			endpoints = append(endpoints, d)
+		}
+	}
+
+	for _, d := range endpoints {
 		if !filepath.IsAbs(d) {
 			return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
 		}
 	}
-	return drives, nil
+	return endpoints, nil
+}
+
+// Parses all arguments and returns a slice of drive paths following the ellipses pattern.
+func parseCacheDrivePaths(arg string) (ep []string, err error) {
+	patterns, perr := ellipses.FindEllipsesPatterns(arg)
+	if perr != nil {
+		return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
+	}
+
+	for _, lbls := range patterns.Expand() {
+		ep = append(ep, strings.Join(lbls, ""))
+	}
+
+	return ep, nil
 }
 
 // Parses given cacheExcludesEnv and returns a list of cache exclude patterns.

cmd/disk-cache-config_test.go

@@ -41,12 +41,32 @@ func TestParseCacheDrives(t *testing.T) {
 			expectedPatterns []string
 			success          bool
 		}{"C:/home/drive1;C:/home/drive2;C:/home/drive3", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1...3}", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1..3}", []string{}, false})
 	} else {
 		testCases = append(testCases, struct {
 			driveStr         string
 			expectedPatterns []string
 			success          bool
 		}{"/home/drive1;/home/drive2;/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1...3}", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1..3}", []string{}, false})
 	}
 	for i, testCase := range testCases {
 		drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheEnvDelimiter))

cmd/disk-cache-fs.go

@@ -30,7 +30,6 @@ import (
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -258,7 +257,7 @@ func (cfs *cacheFSObjects) IsOnline() bool {
 }
 
 // Caches the object to disk
-func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) error {
+func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) error {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:
@@ -275,7 +274,7 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data
 			return pErr
 		}
 	}
-	_, err := cfs.PutObject(ctx, bucket, object, data, metadata)
+	_, err := cfs.PutObject(ctx, bucket, object, data, metadata, opts)
 	// if err is due to disk being offline , mark cache drive as offline
 	if IsErr(err, baseErrs...) {
 		cfs.setOnline(false)
@@ -284,8 +283,8 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data
 }
 
 // Returns the handle for the cached object
-func (cfs *cacheFSObjects) Get(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
-	return cfs.GetObject(ctx, bucket, object, startOffset, length, writer, etag)
+func (cfs *cacheFSObjects) Get(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
+	return cfs.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
 }
 
 // Deletes the cached object
@@ -295,13 +294,14 @@ func (cfs *cacheFSObjects) Delete(ctx context.Context, bucket, object string) (e
 
 // convenience function to check if object is cached on this cacheFSObjects
 func (cfs *cacheFSObjects) Exists(ctx context.Context, bucket, object string) bool {
-	_, err := cfs.GetObjectInfo(ctx, bucket, object)
+	_, err := cfs.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	return err == nil
 }
 
 // Identical to fs PutObject operation except that it uses ETag in metadata
 // headers.
-func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
+func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+	data := r.Reader
 	fs := cfs.FSObjects
 	// Lock the object.
 	objectLock := fs.nsMutex.NewNSLock(bucket, object)
@@ -354,7 +354,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	}
 
 	// Validate input data size and it can never be less than zero.
-	if data.Size() < 0 {
+	if data.Size() < -1 {
 		logger.LogIf(ctx, errInvalidArgument)
 		return ObjectInfo{}, errInvalidArgument
 	}
@@ -438,7 +438,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 // Implements S3 compatible initiate multipart API. Operation here is identical
 // to fs backend implementation - with the exception that cache FS uses the uploadID
 // generated on the backend
-func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, uploadID string) (string, error) {
+func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, uploadID string, opts ObjectOptions) (string, error) {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:

cmd/disk-cache.go

@@ -31,20 +31,15 @@ import (
 
 	"github.com/djherbis/atime"
 
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/wildcard"
-
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/wildcard"
 )
 
-// list of all errors that can be ignored in tree walk operation in disk cache
-var cacheTreeWalkIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errVolumeNotFound, errFileNotFound)
-
 const (
-	// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
-	cacheSizeMultiplier  = 100
+	// disk cache needs to have  object size space free for a cache entry to be created.
 	cacheTrashDir        = "trash"
-	cacheMaxDiskUsagePct = 80 // in %
 	cacheCleanupInterval = 10 // in minutes
 )
 
@@ -62,18 +57,19 @@ type cacheObjects struct {
 	// file path patterns to exclude from cache
 	exclude []string
 	// Object functions pointing to the corresponding functions of backend implementation.
-	GetObjectFn               func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error)
-	GetObjectInfoFn           func(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error)
-	PutObjectFn               func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error)
+	GetObjectNInfoFn          func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
+	GetObjectFn               func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	GetObjectInfoFn           func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObjectFn               func(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn            func(ctx context.Context, bucket, object string) error
 	ListObjectsFn             func(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2Fn           func(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
 	ListBucketsFn             func(ctx context.Context) (buckets []BucketInfo, err error)
 	GetBucketInfoFn           func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
-	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error)
-	PutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error)
+	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	PutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUploadFn    func(ctx context.Context, bucket, object, uploadID string) error
-	CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
+	CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteBucketFn            func(ctx context.Context, bucket string) error
 }
 
@@ -93,21 +89,27 @@ type CacheObjectLayer interface {
 	ListBuckets(ctx context.Context) (buckets []BucketInfo, err error)
 	DeleteBucket(ctx context.Context, bucket string) error
 	// Object operations.
-	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error)
-	GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error)
-	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error)
+	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
+	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
 
 	// Multipart operations.
-	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error)
-	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error)
+	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error
-	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
+	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
 
 	// Storage operations.
 	StorageInfo(ctx context.Context) CacheStorageInfo
 }
 
+// IsCacheable returns if the object should be saved in the cache.
+func (o ObjectInfo) IsCacheable() bool {
+	return !crypto.IsEncrypted(o.UserDefined)
+}
+
 // backendDownError returns true if err is due to backend failure or faulty disk if in server mode
 func backendDownError(err error) bool {
 	_, backendDown := err.(BackendDown)
@@ -180,22 +182,99 @@ func (c cacheObjects) getMetadata(objInfo ObjectInfo) map[string]string {
 	return metadata
 }
 
+func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+	if c.isCacheExclude(bucket, object) {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	// fetch cacheFSObjects if object is currently cached or nearest available cache drive
+	dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object)
+	if err != nil {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	cacheReader, cacheErr := dcache.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+
+	objInfo, err := c.GetObjectInfoFn(ctx, bucket, object, opts)
+	if backendDownError(err) && cacheErr == nil {
+		return cacheReader, nil
+	} else if err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			// Delete cached entry if backend object was deleted.
+			dcache.Delete(ctx, bucket, object)
+		}
+		return nil, err
+	}
+
+	if !objInfo.IsCacheable() || filterFromCache(objInfo.UserDefined) {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	if cacheErr == nil {
+		if cacheReader.ObjInfo.ETag == objInfo.ETag && !isStaleCache(objInfo) {
+			// Object is not stale, so serve from cache
+			return cacheReader, nil
+		}
+		cacheReader.Close()
+		// Object is stale, so delete from cache
+		dcache.Delete(ctx, bucket, object)
+	}
+
+	// Since we got here, we are serving the request from backend,
+	// and also adding the object to the cache.
+
+	if rs != nil {
+		// We don't cache partial objects.
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+	if !dcache.diskAvailable(objInfo.Size) {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	if bkErr != nil {
+		return nil, bkErr
+	}
+
+	// Initialize pipe.
+	pipeReader, pipeWriter := io.Pipe()
+	teeReader := io.TeeReader(bkReader, pipeWriter)
+	hashReader, herr := hash.NewReader(pipeReader, bkReader.ObjInfo.Size, "", "", bkReader.ObjInfo.Size)
+	if herr != nil {
+		bkReader.Close()
+		return nil, herr
+	}
+
+	go func() {
+		opts := ObjectOptions{}
+		putErr := dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), c.getMetadata(bkReader.ObjInfo), opts)
+		// close the write end of the pipe, so the error gets
+		// propagated to getObjReader
+		pipeWriter.CloseWithError(putErr)
+	}()
+
+	cleanupBackend := func() { bkReader.Close() }
+	cleanupPipe := func() { pipeReader.Close() }
+	gr = NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, cleanupBackend, cleanupPipe)
+	return gr, nil
+}
+
 // Uses cached-object to serve the request. If object is not cached it serves the request from the backend and also
 // stores it in the cache for serving subsequent requests.
-func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
+func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
 	GetObjectFn := c.GetObjectFn
 	GetObjectInfoFn := c.GetObjectInfoFn
 
 	if c.isCacheExclude(bucket, object) {
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// fetch cacheFSObjects if object is currently cached or nearest available cache drive
 	dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object)
 	if err != nil {
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// stat object on backend
-	objInfo, err := GetObjectInfoFn(ctx, bucket, object)
+	objInfo, err := GetObjectInfoFn(ctx, bucket, object, opts)
 	backendDown := backendDownError(err)
 	if err != nil && !backendDown {
 		if _, ok := err.(ObjectNotFound); ok {
@@ -205,43 +284,43 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		return err
 	}
 
-	if !backendDown && filterFromCache(objInfo.UserDefined) {
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+	if !backendDown && !objInfo.IsCacheable() {
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 
-	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object)
+	if !backendDown && filterFromCache(objInfo.UserDefined) {
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
+	}
+
+	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object, opts)
 	if err == nil {
 		if backendDown {
 			// If the backend is down, serve the request from cache.
-			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag)
+			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag, opts)
 		}
 		if cachedObjInfo.ETag == objInfo.ETag && !isStaleCache(objInfo) {
-			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag)
+			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag, opts)
 		}
 		dcache.Delete(ctx, bucket, object)
 	}
-	if startOffset != 0 || length != objInfo.Size {
+	if startOffset != 0 || (length > 0 && length != objInfo.Size) {
 		// We don't cache partial objects.
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
-	if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) {
-		// cache only objects < 1/100th of disk capacity
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+	if !dcache.diskAvailable(objInfo.Size) {
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, objInfo.Size, "", "")
+	hashReader, err := hash.NewReader(pipeReader, objInfo.Size, "", "", objInfo.Size)
 	if err != nil {
 		return err
 	}
 	go func() {
-		if err = GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag); err != nil {
-			pipeWriter.CloseWithError(err)
-			return
-		}
-		pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
+		gerr := GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag, opts)
+		pipeWriter.CloseWithError(gerr) // Close writer explicitly signaling we wrote all data.
 	}()
-	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo))
+	err = dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), c.getMetadata(objInfo), opts)
 	if err != nil {
 		return err
 	}
@@ -250,17 +329,17 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 }
 
 // Returns ObjectInfo from cache if available.
-func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string) (ObjectInfo, error) {
+func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 	getObjectInfoFn := c.GetObjectInfoFn
 	if c.isCacheExclude(bucket, object) {
-		return getObjectInfoFn(ctx, bucket, object)
+		return getObjectInfoFn(ctx, bucket, object, opts)
 	}
 	// fetch cacheFSObjects if object is currently cached or nearest available cache drive
 	dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object)
 	if err != nil {
-		return getObjectInfoFn(ctx, bucket, object)
+		return getObjectInfoFn(ctx, bucket, object, opts)
 	}
-	objInfo, err := getObjectInfoFn(ctx, bucket, object)
+	objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
 	if err != nil {
 		if _, ok := err.(ObjectNotFound); ok {
 			// Delete the cached entry if backend object was deleted.
@@ -271,14 +350,14 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string)
 			return ObjectInfo{}, err
 		}
 		// when backend is down, serve from cache.
-		cachedObjInfo, cerr := dcache.GetObjectInfo(ctx, bucket, object)
+		cachedObjInfo, cerr := dcache.GetObjectInfo(ctx, bucket, object, opts)
 		if cerr == nil {
 			return cachedObjInfo, nil
 		}
 		return ObjectInfo{}, BackendDown{}
 	}
 	// when backend is up, do a sanity check on cached object
-	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object)
+	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		return objInfo, nil
 	}
@@ -292,40 +371,32 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string)
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - list of fsObjects
-func listDirCacheFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks []*cacheFSObjects) listDirFunc {
-	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string, err error) {
+func listDirCacheFactory(isLeaf isLeafFunc, disks []*cacheFSObjects) listDirFunc {
+	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string) {
 		var entries []string
 		for _, disk := range disks {
 			// ignore disk-caches that might be missing/offline
 			if disk == nil {
 				continue
 			}
-			fs := disk.FSObjects
-			entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 
-			// For any reason disk was deleted or goes offline, continue
-			// and list from other disks if possible.
+			fs := disk.FSObjects
+			var err error
+			entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 			if err != nil {
-				if IsErrIgnored(err, treeWalkIgnoredErrs...) {
-					continue
-				}
-				return nil, err
+				continue
 			}
 
 			// Filter entries that have the prefix prefixEntry.
 			entries = filterMatchingPrefix(entries, prefixEntry)
 			dirs = append(dirs, entries...)
 		}
-		return dirs, nil
+		return dirs
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
-		var cacheEntries []string
-		cacheEntries, err = listCacheDirs(bucket, prefixDir, prefixEntry)
-		if err != nil {
-			return nil, false, err
-		}
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
+		cacheEntries := listCacheDirs(bucket, prefixDir, prefixEntry)
 		for _, entry := range cacheEntries {
 			// Find elements in entries which are not in mergedEntries
 			idx := sort.SearchStrings(mergedEntries, entry)
@@ -336,7 +407,7 @@ func listDirCacheFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks [
 			mergedEntries = append(mergedEntries, entry)
 			sort.Strings(mergedEntries)
 		}
-		return mergedEntries, false, nil
+		return mergedEntries, false
 	}
 	return listDir
 }
@@ -372,7 +443,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			return fs.isObjectDir(bucket, object)
 		}
 
-		listDir := listDirCacheFactory(isLeaf, cacheTreeWalkIgnoredErrs, c.cache.cfs)
+		listDir := listDirCacheFactory(isLeaf, c.cache.cfs)
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 
@@ -573,42 +644,43 @@ func (c cacheObjects) isCacheExclude(bucket, object string) bool {
 }
 
 // PutObject - caches the uploaded object for single Put operations
-func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
+func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	putObjectFn := c.PutObjectFn
+	data := r.Reader
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return putObjectFn(ctx, bucket, object, r, metadata)
+		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	size := r.Size()
 
 	// fetch from backend if there is no space on cache drive
-	if !dcache.diskAvailable(size * cacheSizeMultiplier) {
-		return putObjectFn(ctx, bucket, object, r, metadata)
+	if !dcache.diskAvailable(size) {
+		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	// fetch from backend if cache exclude pattern or cache-control
 	// directive set to exclude
 	if c.isCacheExclude(bucket, object) || filterFromCache(metadata) {
 		dcache.Delete(ctx, bucket, object)
-		return putObjectFn(ctx, bucket, object, r, metadata)
+		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	objInfo = ObjectInfo{}
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, r.MD5HexString(), r.SHA256HexString())
+	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	// Initialize pipe to stream data to cache
 	rPipe, wPipe := io.Pipe()
-	cHashReader, err := hash.NewReader(rPipe, size, r.MD5HexString(), r.SHA256HexString())
+	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	oinfoCh := make(chan ObjectInfo)
 	errCh := make(chan error)
 	go func() {
-		oinfo, perr := putObjectFn(ctx, bucket, object, hashReader, metadata)
+		oinfo, perr := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), metadata, opts)
 		if perr != nil {
 			pipeWriter.CloseWithError(perr)
 			wPipe.CloseWithError(perr)
@@ -621,14 +693,14 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 	}()
 
 	go func() {
-		if err = dcache.Put(ctx, bucket, object, cHashReader, metadata); err != nil {
+		if err = dcache.Put(ctx, bucket, object, NewPutObjReader(cHashReader, nil, nil), metadata, opts); err != nil {
 			wPipe.CloseWithError(err)
 			return
 		}
 	}()
 
 	mwriter := io.MultiWriter(pipeWriter, wPipe)
-	_, err = io.Copy(mwriter, r)
+	_, err = io.Copy(mwriter, data)
 	if err != nil {
 		err = <-errCh
 		return objInfo, err
@@ -640,68 +712,69 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 }
 
 // NewMultipartUpload - Starts a new multipart upload operation to backend and cache.
-func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
 	newMultipartUploadFn := c.NewMultipartUploadFn
 
 	if c.isCacheExclude(bucket, object) || filterFromCache(metadata) {
-		return newMultipartUploadFn(ctx, bucket, object, metadata)
+		return newMultipartUploadFn(ctx, bucket, object, metadata, opts)
 	}
 
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return newMultipartUploadFn(ctx, bucket, object, metadata)
+		return newMultipartUploadFn(ctx, bucket, object, metadata, opts)
 	}
 
-	uploadID, err = newMultipartUploadFn(ctx, bucket, object, metadata)
+	uploadID, err = newMultipartUploadFn(ctx, bucket, object, metadata, opts)
 	if err != nil {
 		return
 	}
 	// create new multipart upload in cache with same uploadID
-	dcache.NewMultipartUpload(ctx, bucket, object, metadata, uploadID)
+	dcache.NewMultipartUpload(ctx, bucket, object, metadata, uploadID, opts)
 	return uploadID, err
 }
 
 // PutObjectPart - uploads part to backend and cache simultaneously.
-func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
+func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
+	data := r.Reader
 	putObjectPartFn := c.PutObjectPartFn
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, r, opts)
 	}
 
 	if c.isCacheExclude(bucket, object) {
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, r, opts)
 	}
 
-	// make sure cache has at least cacheSizeMultiplier * size available
+	// make sure cache has at least size space available
 	size := data.Size()
-	if !dcache.diskAvailable(size * cacheSizeMultiplier) {
+	if !dcache.diskAvailable(size) {
 		select {
 		case dcache.purgeChan <- struct{}{}:
 		default:
 		}
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, r, opts)
 	}
 
 	info = PartInfo{}
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString())
+	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
 	if err != nil {
 		return
 	}
 	// Initialize pipe to stream data to cache
 	rPipe, wPipe := io.Pipe()
-	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString())
+	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
 	if err != nil {
 		return
 	}
 	pinfoCh := make(chan PartInfo)
 	errorCh := make(chan error)
 	go func() {
-		info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, hashReader)
+		info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, NewPutObjReader(hashReader, nil, nil), opts)
 		if err != nil {
 			close(pinfoCh)
 			pipeWriter.CloseWithError(err)
@@ -713,7 +786,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 		pinfoCh <- info
 	}()
 	go func() {
-		if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, cHashReader); perr != nil {
+		if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, NewPutObjReader(cHashReader, nil, nil), opts); perr != nil {
 			wPipe.CloseWithError(perr)
 			return
 		}
@@ -755,25 +828,25 @@ func (c cacheObjects) AbortMultipartUpload(ctx context.Context, bucket, object,
 }
 
 // CompleteMultipartUpload - completes multipart upload operation on backend and cache.
-func (c cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
+func (c cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	completeMultipartUploadFn := c.CompleteMultipartUploadFn
 
 	if c.isCacheExclude(bucket, object) {
-		return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts)
+		return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
 
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts)
+		return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
 	// perform backend operation
-	objInfo, err = completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts)
+	objInfo, err = completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
 	if err != nil {
 		return
 	}
 	// create new multipart upload in cache with same uploadID
-	dcache.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts)
+	dcache.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 	return
 }
 
@@ -839,7 +912,7 @@ func newCache(config CacheConfig) (*diskCache, error) {
 		if err := checkAtimeSupport(dir); err != nil {
 			return nil, errors.New("Atime support required for disk caching")
 		}
-		cache, err := newCacheFSObjects(dir, config.Expiry, cacheMaxDiskUsagePct)
+		cache, err := newCacheFSObjects(dir, config.Expiry, config.MaxUse)
 		if err != nil {
 			return nil, err
 		}
@@ -889,14 +962,17 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		cache:    dcache,
 		exclude:  config.Exclude,
 		listPool: newTreeWalkPool(globalLookupTimeout),
-		GetObjectFn: func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
-			return newObjectLayerFn().GetObject(ctx, bucket, object, startOffset, length, writer, etag)
+		GetObjectFn: func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
+			return newObjectLayerFn().GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
 		},
-		GetObjectInfoFn: func(ctx context.Context, bucket, object string) (ObjectInfo, error) {
-			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object)
+		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},
-		PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
-			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata)
+		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+		},
+		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata, opts)
 		},
 		DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
 			return newObjectLayerFn().DeleteObject(ctx, bucket, object)
@@ -913,17 +989,17 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		GetBucketInfoFn: func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
 			return newObjectLayerFn().GetBucketInfo(ctx, bucket)
 		},
-		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
-			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata)
+		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
+			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata, opts)
 		},
-		PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
-			return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data)
+		PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
+			return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 		},
 		AbortMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string) error {
 			return newObjectLayerFn().AbortMultipartUpload(ctx, bucket, object, uploadID)
 		},
-		CompleteMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
-			return newObjectLayerFn().CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts)
+		CompleteMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 		},
 		DeleteBucketFn: func(ctx context.Context, bucket string) error {
 			return newObjectLayerFn().DeleteBucket(ctx, bucket)

cmd/disk-cache_test.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"os"
 	"reflect"
 	"testing"
 	"time"
@@ -28,21 +27,15 @@ import (
 )
 
 // Initialize cache FS objects.
-func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
-	newTestConfig(globalMinioDefaultRegion)
-	var err error
-	obj, err := newCacheFSObjects(disk, globalCacheExpiry, 100)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return obj, nil
+func initCacheFSObjects(disk string, cacheMaxUse int) (*cacheFSObjects, error) {
+	return newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
 }
 
 // inits diskCache struct for nDisks
-func initDiskCaches(drives []string, t *testing.T) (*diskCache, error) {
+func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
 	var cfs []*cacheFSObjects
 	for _, d := range drives {
-		obj, err := initCacheFSObjects(d, t)
+		obj, err := initCacheFSObjects(d, cacheMaxUse)
 		if err != nil {
 			return nil, err
 		}
@@ -59,7 +52,46 @@ func TestGetCacheFS(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, t)
+		d, err := initDiskCaches(fsDirs, 100, t)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bucketName := "testbucket"
+		objectName := "testobject"
+		ctx := context.Background()
+		// find cache drive where object would be hashed
+		index := d.hashIndex(bucketName, objectName)
+		// turn off drive by setting online status to false
+		d.cfs[index].online = false
+		cfs, err := d.getCacheFS(ctx, bucketName, objectName)
+		if n == 1 && err == errDiskNotFound {
+			continue
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		i := -1
+		for j, f := range d.cfs {
+			if f == cfs {
+				i = j
+				break
+			}
+		}
+		if i != (index+1)%n {
+			t.Fatalf("expected next cache location to be picked")
+		}
+	}
+}
+
+// test whether a drive being offline causes
+// getCacheFS to fetch next online drive
+func TestGetCacheFSMaxUse(t *testing.T) {
+	for n := 1; n < 10; n++ {
+		fsDirs, err := getRandomDisks(n)
+		if err != nil {
+			t.Fatal(err)
+		}
+		d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -92,11 +124,6 @@ func TestGetCacheFS(t *testing.T) {
 
 // test wildcard patterns for excluding entries from cache
 func TestCacheExclusion(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
 	fsDirs, err := getRandomDisks(1)
 	if err != nil {
 		t.Fatal(err)
@@ -141,7 +168,7 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, t)
+	d, err := initDiskCaches(fsDirs, 100, t)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -165,17 +192,18 @@ func TestDiskCache(t *testing.T) {
 	objInfo.ContentType = contentType
 	objInfo.ETag = etag
 	objInfo.UserDefined = httpMeta
+	opts := ObjectOptions{}
 
 	byteReader := bytes.NewReader([]byte(content))
-	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+	err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
-	cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName)
+	cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -192,7 +220,7 @@ func TestDiskCache(t *testing.T) {
 		t.Fatal("Cached content-type does not match")
 	}
 	writer := bytes.NewBuffer(nil)
-	err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "")
+	err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -209,6 +237,88 @@ func TestDiskCache(t *testing.T) {
 	}
 }
 
+// Test diskCache with upper bound on max cache use.
+func TestDiskCacheMaxUse(t *testing.T) {
+	fsDirs, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cache := d.cfs[0]
+	ctx := context.Background()
+	bucketName := "testbucket"
+	objectName := "testobject"
+	content := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	etag := "061208c10af71a30c6dcd6cf5d89f0fe"
+	contentType := "application/zip"
+	size := len(content)
+
+	httpMeta := make(map[string]string)
+	httpMeta["etag"] = etag
+	httpMeta["content-type"] = contentType
+
+	objInfo := ObjectInfo{}
+	objInfo.Bucket = bucketName
+	objInfo.Name = objectName
+	objInfo.Size = int64(size)
+	objInfo.ContentType = contentType
+	objInfo.ETag = etag
+	objInfo.UserDefined = httpMeta
+	opts := ObjectOptions{}
+
+	byteReader := bytes.NewReader([]byte(content))
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !cache.diskAvailable(int64(size)) {
+		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
+		if err != errDiskFull {
+			t.Fatal("Cache max-use limit violated.")
+		}
+	} else {
+		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
+		if err != nil {
+			t.Fatal(err)
+		}
+		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !cache.Exists(ctx, bucketName, objectName) {
+			t.Fatal("Expected object to exist on cache")
+		}
+		if cachedObjInfo.ETag != objInfo.ETag {
+			t.Fatal("Expected ETag to match")
+		}
+		if cachedObjInfo.Size != objInfo.Size {
+			t.Fatal("Size mismatch")
+		}
+		if cachedObjInfo.ContentType != objInfo.ContentType {
+			t.Fatal("Cached content-type does not match")
+		}
+		writer := bytes.NewBuffer(nil)
+		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
+			t.Errorf("wrong cached file content")
+		}
+		err = cache.Delete(ctx, bucketName, objectName)
+		if err != nil {
+			t.Errorf("object missing from cache")
+		}
+		online := cache.IsOnline()
+		if !online {
+			t.Errorf("expected cache drive to be online")
+		}
+	}
+}
+
 func TestIsCacheExcludeDirective(t *testing.T) {
 	testCases := []struct {
 		cacheControlOpt string

cmd/dummy-data-generator_test.go

@@ -0,0 +1,182 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+var alphabets = []byte("abcdefghijklmnopqrstuvwxyz0123456789")
+
+// DummyDataGen returns a reader that repeats the bytes in `alphabets`
+// upto the desired length.
+type DummyDataGen struct {
+	b           []byte
+	idx, length int64
+}
+
+// NewDummyDataGen returns a ReadSeeker over the first `totalLength`
+// bytes from the infinite stream consisting of repeated
+// concatenations of `alphabets`.
+//
+// The skipOffset (generally = 0) can be used to skip a given number
+// of bytes from the beginning of the infinite stream. This is useful
+// to compare such streams of bytes that may be split up, because:
+//
+// Given the function:
+//
+// f := func(r io.Reader) string {
+//           b, _ := ioutil.ReadAll(r)
+//           return string(b)
+// }
+//
+// for example, the following is true:
+//
+// f(NewDummyDataGen(100, 0)) == f(NewDummyDataGen(50, 0)) + f(NewDummyDataGen(50, 50))
+func NewDummyDataGen(totalLength, skipOffset int64) io.ReadSeeker {
+	if totalLength < 0 {
+		panic("Negative length passed to DummyDataGen!")
+	}
+	if skipOffset < 0 {
+		panic("Negative rotations are not allowed")
+	}
+
+	skipOffset = skipOffset % int64(len(alphabets))
+	as := make([]byte, 2*len(alphabets))
+	copy(as, alphabets)
+	copy(as[len(alphabets):], alphabets)
+	b := as[skipOffset : skipOffset+int64(len(alphabets))]
+	return &DummyDataGen{
+		length: totalLength,
+		b:      b,
+	}
+}
+
+func (d *DummyDataGen) Read(b []byte) (n int, err error) {
+	k := len(b)
+	numLetters := int64(len(d.b))
+	for k > 0 && d.idx < d.length {
+		w := copy(b[len(b)-k:], d.b[d.idx%numLetters:])
+		k -= w
+		d.idx += int64(w)
+		n += w
+	}
+	if d.idx >= d.length {
+		extraBytes := d.idx - d.length
+		n -= int(extraBytes)
+		if n < 0 {
+			n = 0
+		}
+		err = io.EOF
+	}
+	return
+}
+
+func (d *DummyDataGen) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekStart:
+		if offset < 0 {
+			return 0, errors.New("Invalid offset")
+		}
+		d.idx = offset
+	case io.SeekCurrent:
+		if d.idx+offset < 0 {
+			return 0, errors.New("Invalid offset")
+		}
+		d.idx += offset
+	case io.SeekEnd:
+		if d.length+offset < 0 {
+			return 0, errors.New("Invalid offset")
+		}
+		d.idx = d.length + offset
+	}
+	return d.idx, nil
+}
+
+func TestDummyDataGenerator(t *testing.T) {
+	readAll := func(r io.Reader) string {
+		b, _ := ioutil.ReadAll(r)
+		return string(b)
+	}
+	checkEq := func(a, b string) {
+		if a != b {
+			t.Fatalf("Unexpected equality failure")
+		}
+	}
+
+	checkEq(readAll(NewDummyDataGen(0, 0)), "")
+
+	checkEq(readAll(NewDummyDataGen(10, 0)), readAll(NewDummyDataGen(10, int64(len(alphabets)))))
+
+	checkEq(readAll(NewDummyDataGen(100, 0)), readAll(NewDummyDataGen(50, 0))+readAll(NewDummyDataGen(50, 50)))
+
+	r := NewDummyDataGen(100, 0)
+	r.Seek(int64(len(alphabets)), 0)
+	checkEq(readAll(r), readAll(NewDummyDataGen(100-int64(len(alphabets)), 0)))
+}
+
+// Compares all the bytes returned by the given readers. Any Read
+// errors cause a `false` result. A string describing the error is
+// also returned.
+func cmpReaders(r1, r2 io.Reader) (bool, string) {
+	bufLen := 32 * 1024
+	b1, b2 := make([]byte, bufLen), make([]byte, bufLen)
+	for i := 0; true; i++ {
+		n1, e1 := io.ReadFull(r1, b1)
+		n2, e2 := io.ReadFull(r2, b2)
+		if n1 != n2 {
+			return false, fmt.Sprintf("Read %d != %d bytes from the readers", n1, n2)
+		}
+		if !bytes.Equal(b1[:n1], b2[:n2]) {
+			return false, fmt.Sprintf("After reading %d equal buffers (32Kib each), we got the following two strings:\n%v\n%v\n",
+				i, b1, b2)
+		}
+		// Check if stream has ended
+		if (e1 == io.ErrUnexpectedEOF && e2 == io.ErrUnexpectedEOF) || (e1 == io.EOF && e2 == io.EOF) {
+			break
+		}
+		if e1 != nil || e2 != nil {
+			return false, fmt.Sprintf("Got unexpected error values: %v == %v", e1, e2)
+		}
+	}
+	return true, ""
+}
+
+func TestCmpReaders(t *testing.T) {
+	{
+		r1 := bytes.NewBuffer([]byte("abc"))
+		r2 := bytes.NewBuffer([]byte("abc"))
+		ok, msg := cmpReaders(r1, r2)
+		if !(ok && msg == "") {
+			t.Fatalf("unexpected")
+		}
+	}
+
+	{
+		r1 := bytes.NewBuffer([]byte("abc"))
+		r2 := bytes.NewBuffer([]byte("abcd"))
+		ok, _ := cmpReaders(r1, r2)
+		if ok {
+			t.Fatalf("unexpected")
+		}
+	}
+}

cmd/dummy-object-layer_test.go

@@ -19,9 +19,8 @@ package cmd
 import (
 	"context"
 	"io"
-	"time"
+	"net/http"
 
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -60,19 +59,23 @@ func (api *DummyObjectLayer) ListObjectsV2(ctx context.Context, bucket, prefix,
 	return
 }
 
-func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
+func (api *DummyObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lock LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
 	return
 }
 
-func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return
 }
 
@@ -84,15 +87,15 @@ func (api *DummyObjectLayer) ListMultipartUploads(ctx context.Context, bucket, p
 	return
 }
 
-func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo) (info PartInfo, err error) {
+func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
+func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
 	return
 }
 
@@ -104,7 +107,7 @@ func (api *DummyObjectLayer) AbortMultipartUpload(ctx context.Context, bucket, o
 	return
 }
 
-func (api *DummyObjectLayer) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return
 }
 
@@ -132,14 +135,6 @@ func (api *DummyObjectLayer) ListObjectsHeal(ctx context.Context, bucket, prefix
 	return
 }
 
-func (api *DummyObjectLayer) ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) (info []VolumeLockInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ClearLocks(context.Context, []VolumeLockInfo) (err error) {
-	return
-}
-
 func (api *DummyObjectLayer) SetBucketPolicy(context.Context, string, *policy.Policy) (err error) {
 	return
 }
@@ -163,3 +158,7 @@ func (api *DummyObjectLayer) IsNotificationSupported() (b bool) {
 func (api *DummyObjectLayer) IsEncryptionSupported() (b bool) {
 	return
 }
+
+func (api *DummyObjectLayer) IsCompressionSupported() (b bool) {
+	return
+}