docs: fix all Chinese doc links for the new docs site (#6097)

Additionally fix typos, default to US locale words

.travis.yml

@@ -31,4 +31,4 @@ after_success:
 - bash <(curl -s https://codecov.io/bash)
 
 go:
-- 1.9.4
+- '1.10.1'

CONTRIBUTING.md

@@ -6,7 +6,7 @@
 
 Start by forking the Minio GitHub repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details:
 
-### Setup your Minio Github Repository
+### Setup your Minio GitHub Repository
 Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL of your Minio fork (you will need it for the `git clone` command below).
 
 ```sh

Dockerfile

@@ -1,6 +1,6 @@
-FROM golang:1.9.4-alpine3.6
+FROM golang:1.10.1-alpine3.7
 
-MAINTAINER Minio Inc <dev@minio.io>
+LABEL maintainer="Minio Inc <dev@minio.io>"
 
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin

Dockerfile.dev

@@ -1,6 +1,6 @@
-FROM golang:1.9.4-alpine3.6
+FROM golang:1.10.1-alpine3.7
 
-MAINTAINER Minio Inc <dev@minio.io>
+LABEL maintainer="Minio Inc <dev@minio.io>"
 
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin

Dockerfile.release

@@ -1,7 +1,7 @@
-FROM alpine:3.6
+FROM alpine:3.7
+
+LABEL maintainer="Minio Inc <dev@minio.io>"
 
-MAINTAINER Minio Inc <dev@minio.io>
- 
 COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 
 ENV MINIO_UPDATE off
@@ -27,4 +27,3 @@ HEALTHCHECK --interval=30s --timeout=5s \
     CMD /usr/bin/healthcheck.sh
 
 CMD ["minio"]
-

MAINTAINERS.md

@@ -1,6 +1,6 @@
 # For maintainers only
 
-### Setup your minio Github Repository
+### Setup your minio GitHub Repository
 
 Fork [minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository.
 ```bash

Makefile

@@ -50,15 +50,17 @@ deadcode:
 	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
 
 spelling:
-	@${GOPATH}/bin/misspell -error `find cmd/`
-	@${GOPATH}/bin/misspell -error `find pkg/`
-	@${GOPATH}/bin/misspell -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find cmd/`
+	@${GOPATH}/bin/misspell -locale US -error `find pkg/`
+	@${GOPATH}/bin/misspell -locale US -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
+	@${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build
 	@echo "Running unit tests"
-	@go test $(GOFLAGS) ./...
+	@go test $(GOFLAGS) -tags kqueue ./...
 	@echo "Verifying build"
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
@@ -69,7 +71,7 @@ coverage: build
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@CGO_ENABLED=0 go build --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
+	@CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
 
 pkg-add:
 	@echo "Adding new package $(PKG)"
@@ -89,7 +91,7 @@ pkg-list:
 # Builds minio and installs it to $GOPATH/bin.
 install: build
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
-	@cp $(PWD)/minio $(GOPATH)/bin/minio
+	@mkdir -p $(GOPATH)/bin && cp $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
 clean:

README.md

@@ -101,7 +101,7 @@ iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
 service iptables restart
 ```
 
-### ufw in Debian
+### ufw
 
 For hosts with ufw enabled (Debian based distros), you can use `ufw` command to allow traffic to specific ports. Use below command to allow access to port 9000
 
@@ -115,10 +115,30 @@ Below command enables all incoming traffic to ports ranging from 9000 to 9010.
 ufw allow 9000:9010/tcp
 ```
 
+### firewall-cmd
+
+For hosts with firewall-cmd enabled (CentOS), you can use `firewall-cmd` command to allow traffic to specific ports. Use below commands to allow access to port 9000
+
+```sh
+firewall-cmd --get-active-zones
+```
+
+This command gets the active zone(s). Now, apply port rules to the relevant zones returned above. For example if the zone is `public`, use
+
+```sh
+firewall-cmd --zone=public --add-port=9000/tcp --permanent
+```
+
+Note that `permanent` makes sure the rules are persistent across firewall start, restart or reload. Finally reload the firewall for changes to take effect.
+
+```sh
+firewall-cmd --reload
+```
+
 ## Test using Minio Browser
 Minio Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully.
 
-![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true)
+![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.png?raw=true)
 
 ## Test using Minio Client `mc`
 `mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the Minio Client [Quickstart Guide](https://docs.minio.io/docs/minio-client-quickstart-guide) for further instructions.

README_zh_CN.md

@@ -84,7 +84,7 @@ go get -u github.com/minio/minio
 ## 使用Minio浏览器进行验证
 安装后使用浏览器访问[http://127.0.0.1:9000](http://127.0.0.1:9000)，如果可以访问，则表示minio已经安装成功。
 
-![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true)
+![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.png?raw=true)
 
 ## 使用Minio客户端 `mc`进行验证
 `mc` 提供了一些UNIX常用命令的替代品，像ls, cat, cp, mirror, diff这些。 它支持文件系统和亚马逊S3云存储服务。 更多信息请参考 [mc快速入门](https://docs.minio.io/docs/minio-client-quickstart-guide) 。

appveyor.yml

@@ -12,7 +12,7 @@ clone_folder: c:\gopath\src\github.com\minio\minio
 # Environment variables
 environment:
   GOPATH: c:\gopath
-  GOROOT: c:\go19
+  GOROOT: c:\go
 
 # scripts that run after cloning repository
 install:

browser/app/js/browser/BrowserDropdown.js

@@ -89,7 +89,7 @@ export class BrowserDropdown extends React.Component {
           <Dropdown.Menu className="dropdown-menu-right">
             <li>
               <a target="_blank" href="https://github.com/minio/minio">
-                Github <i className="fa fa-github" />
+                GitHub <i className="fa fa-github" />
               </a>
             </li>
             <li>

browser/app/js/browser/ChangePasswordModal.js

@@ -42,7 +42,7 @@ export class ChangePasswordModal extends React.Component {
     const { serverInfo } = this.props
 
     // Check environment variables first.
-    if (serverInfo.info.isEnvCreds) {
+    if (serverInfo.info.isEnvCreds || serverInfo.info.isWorm) {
       this.setState({
         accessKey: "xxxxxxxxx",
         secretKey: "xxxxxxxxx",

browser/app/js/browser/Login.js

@@ -25,14 +25,35 @@ import web from "../web"
 import { Redirect } from "react-router-dom"
 
 export class Login extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      accessKey: "",
+      secretKey: ""
+    }
+  }
+
+  // Handle field changes
+  accessKeyChange(e) {
+    this.setState({
+      accessKey: e.target.value
+    })
+  }
+
+  secretKeyChange(e) {
+    this.setState({
+      secretKey: e.target.value
+    })
+  }
+
   handleSubmit(event) {
     event.preventDefault()
     const { showAlert, history } = this.props
     let message = ""
-    if (!document.getElementById("accessKey").value) {
+    if (this.state.accessKey === "") {
       message = "Access Key cannot be empty"
     }
-    if (!document.getElementById("secretKey").value) {
+    if (this.state.secretKey === "") {
       message = "Secret Key cannot be empty"
     }
     if (message) {
@@ -41,8 +62,8 @@ export class Login extends React.Component {
     }
     web
       .Login({
-        username: document.getElementById("accessKey").value,
-        password: document.getElementById("secretKey").value
+        username: this.state.accessKey,
+        password: this.state.secretKey
       })
       .then(res => {
         history.push("/")
@@ -77,6 +98,8 @@ export class Login extends React.Component {
         <div className="l-wrap">
           <form onSubmit={this.handleSubmit.bind(this)}>
             <InputGroup
+              value={this.state.accessKey}
+              onChange={this.accessKeyChange.bind(this)}
               className="ig-dark"
               label="Access Key"
               id="accessKey"
@@ -87,6 +110,8 @@ export class Login extends React.Component {
               autoComplete="username"
             />
             <InputGroup
+              value={this.state.secretKey}
+              onChange={this.secretKeyChange.bind(this)}
               className="ig-dark"
               label="Secret Key"
               id="secretKey"

browser/app/js/browser/MainActions.js

@@ -33,7 +33,11 @@ export const MainActions = ({
   )
   const onFileUpload = e => {
     e.preventDefault()
-    uploadFile(e.target.files[0])
+    let files = e.target.files
+    let filesToUploadCount = files.length
+    for (let i = 0; i < filesToUploadCount; i++) {
+      uploadFile(files.item(i))
+    }
     e.target.value = null
   }
 
@@ -55,6 +59,7 @@ export const MainActions = ({
                 onChange={onFileUpload}
                 style={{ display: "none" }}
                 id="file-input"
+                multiple={true}
               />
               <label htmlFor="file-input">
                 {" "}

browser/app/js/browser/StorageInfo.js

@@ -25,23 +25,16 @@ export class StorageInfo extends React.Component {
     fetchStorageInfo()
   }
   render() {
-    const { total, free } = this.props.storageInfo
-    const used = total - free
-    const usedPercent = used / total * 100 + "%"
-    const freePercent = free * 100 / total
+    const { used } = this.props.storageInfo
     return (
-      <div className="feh-usage">
+      <div className="feh-used">
         <div className="fehu-chart">
-          <div style={{ width: usedPercent }} />
+          <div style={{ width: 0 }} />
         </div>
         <ul>
           <li>
             <span>Used: </span>
-            {humanize.filesize(total - free)}
-          </li>
-          <li className="pull-right">
-            <span>Free: </span>
-            {humanize.filesize(total - used)}
+            {humanize.filesize(used)}
           </li>
         </ul>
       </div>

browser/app/js/browser/__tests__/Login.test.js

@@ -60,20 +60,25 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
     // case where both keys are empty - displays the second warning
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
 
     // case where access key is empty
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Access Key cannot be empty")
 
     // case where secret key is empty
-    document.getElementById("accessKey").value = "accessKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: ""
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
   })
@@ -85,11 +90,12 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
-    document.getElementById("accessKey").value = "accessKey"
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(web.Login).toHaveBeenCalledWith({
       "username": "accessKey", 

browser/app/js/browser/__tests__/MainActions.test.js

@@ -63,11 +63,20 @@ describe("MainActions", () => {
   it("should call uploadFile when a file is selected for upload", () => {
     const uploadFile = jest.fn()
     const wrapper = shallow(<MainActions uploadFile={uploadFile} />)
-    const file = new Blob(["file content"], { type: "text/plain" })
-    wrapper.find("#file-input").simulate("change", {
+    const files = [new Blob(["file content"], { type: "text/plain" })]
+    const input = wrapper.find("#file-input")
+    const event = {
       preventDefault: jest.fn(),
-      target: { files: [file] }
-    })
-    expect(uploadFile).toHaveBeenCalledWith(file)
+      target: {
+        files: {
+          length: files.length,
+          item: function(index) {
+            return files[index]
+          }
+        }
+      }
+    }
+    input.simulate("change", event)
+    expect(uploadFile).toHaveBeenCalledWith(files[0])
   })
 })

browser/app/js/browser/__tests__/StorageInfo.test.js

@@ -22,7 +22,7 @@ describe("StorageInfo", () => {
   it("should render without crashing", () => {
     shallow(
       <StorageInfo
-        storageInfo={{ total: 100, free: 60 }}
+        storageInfo={{ used: 60 }}
         fetchStorageInfo={jest.fn()}
       />
     )
@@ -32,7 +32,7 @@ describe("StorageInfo", () => {
     const fetchStorageInfo = jest.fn()
     shallow(
       <StorageInfo
-        storageInfo={{ total: 100, free: 60 }}
+        storageInfo={{ used: 60 }}
         fetchStorageInfo={fetchStorageInfo}
       />
     )

browser/app/js/browser/__tests__/actions.test.js

@@ -20,7 +20,7 @@ import * as actionsCommon from "../actions"
 
 jest.mock("../../web", () => ({
   StorageInfo: jest.fn(() => {
-    return Promise.resolve({ storageInfo: { Total: 100, Free: 60 } })
+    return Promise.resolve({ storageInfo: { Used: 60 } })
   }),
   ServerInfo: jest.fn(() => {
     return Promise.resolve({
@@ -40,7 +40,7 @@ describe("Common actions", () => {
   it("creates common/SET_STORAGE_INFO after fetching the storage details ", () => {
     const store = mockStore()
     const expectedActions = [
-      { type: "common/SET_STORAGE_INFO", storageInfo: { total: 100, free: 60 } }
+      { type: "common/SET_STORAGE_INFO", storageInfo: { used: 60 } }
     ]
     return store.dispatch(actionsCommon.fetchStorageInfo()).then(() => {
       const actions = store.getActions()

browser/app/js/browser/actions.js

@@ -34,7 +34,7 @@ export const fetchStorageInfo = () => {
     return web.StorageInfo().then(res => {
       const storageInfo = {
         total: res.storageInfo.Total,
-        free: res.storageInfo.Free
+        used:  res.storageInfo.Used
       }
       dispatch(setStorageInfo(storageInfo))
     })

browser/app/js/objects/ObjectContainer.js

@@ -37,7 +37,7 @@ export const ObjectContainer = ({
   if (checkedObjectsCount == 0) {
     props.actionButtons = <ObjectActions object={object} />
   }
-  return <ObjectItem {...props} onClick={() => downloadObject(object.name)} />
+  return <ObjectItem {...props} />
 }
 
 const mapStateToProps = state => {
@@ -46,10 +46,4 @@ const mapStateToProps = state => {
   }
 }
 
-const mapDispatchToProps = dispatch => {
-  return {
-    downloadObject: object => dispatch(actionsObjects.downloadObject(object))
-  }
-}
-
-export default connect(mapStateToProps, mapDispatchToProps)(ObjectContainer)
+export default connect(mapStateToProps)(ObjectContainer)

browser/app/js/objects/ObjectItem.js

@@ -54,7 +54,9 @@ export const ObjectItem = ({
           href="#"
           onClick={e => {
             e.preventDefault()
-            onClick()
+            if (onClick) {
+              onClick()
+            }
           }}
         >
           {name}

browser/app/js/objects/ObjectsBulkActions.js

@@ -28,6 +28,15 @@ export class ObjectsBulkActions extends React.Component {
       showDeleteConfirmation: false
     }
   }
+  handleDownload() {
+    const { checkedObjects, clearChecked, downloadChecked, downloadObject } = this.props
+    if (checkedObjects.length === 1 && !checkedObjects[0].endsWith('/')) {
+      downloadObject(checkedObjects[0])
+      clearChecked()
+    } else {
+      downloadChecked()
+    }
+  }
   deleteChecked() {
     const { deleteChecked } = this.props
     deleteChecked()
@@ -39,24 +48,27 @@ export class ObjectsBulkActions extends React.Component {
     })
   }
   render() {
-    const { checkedObjectsCount, downloadChecked, clearChecked } = this.props
+    const { checkedObjects, clearChecked } = this.props
     return (
       <div
         className={
           "list-actions" +
           classNames({
-            " list-actions-toggled": checkedObjectsCount > 0
+            " list-actions-toggled": checkedObjects.length > 0
           })
         }
       >
         <span className="la-label">
-          <i className="fa fa-check-circle" /> {checkedObjectsCount} Objects
+          <i className="fa fa-check-circle" /> {checkedObjects.length}
+          {checkedObjects.length === 1 ? " Object " : " Objects "}
           selected
         </span>
         <span className="la-actions pull-right">
-          <button id="download-checked" onClick={downloadChecked}>
+          <button id="download-checked" onClick={this.handleDownload.bind(this)}>
             {" "}
-            Download all as zip{" "}
+            Download
+            {(checkedObjects.length === 1 && !checkedObjects[0].endsWith('/')) ? 
+            " object" : " all as zip" }{" "}
           </button>
         </span>
         <span className="la-actions pull-right">
@@ -86,13 +98,16 @@ export class ObjectsBulkActions extends React.Component {
 
 const mapStateToProps = state => {
   return {
-    checkedObjectsCount: getCheckedList(state).length
+    checkedObjects: getCheckedList(state)
   }
 }
 
 const mapDispatchToProps = dispatch => {
   return {
+    downloadObject: object => dispatch(actions.downloadObject(object)),
     downloadChecked: () => dispatch(actions.downloadCheckedObjects()),
+    downloadObject: object => dispatch(actions.downloadObject(object)),
+    resetCheckedList: () => dispatch(actions.resetCheckedList()),
     clearChecked: () => dispatch(actions.resetCheckedList()),
     deleteChecked: () => dispatch(actions.deleteCheckedObjects())
   }

browser/app/js/objects/__tests__/ObjectItem.test.js

@@ -28,9 +28,16 @@ describe("ObjectItem", () => {
     expect(wrapper.prop("data-type")).toBe("image")
   })
 
-  it("should call onClick when the object isclicked", () => {
+  it("shouldn't call onClick when the object isclicked", () => {
     const onClick = jest.fn()
-    const wrapper = shallow(<ObjectItem name={"test"} onClick={onClick} />)
+    const wrapper = shallow(<ObjectItem name={"test"} />)
+    wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
+    expect(onClick).not.toHaveBeenCalled()
+  })
+
+  it("should call onClick when the folder isclicked", () => {
+    const onClick = jest.fn()
+    const wrapper = shallow(<ObjectItem name={"test/"} onClick={onClick} />)
     wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
     expect(onClick).toHaveBeenCalled()
   })

browser/app/js/objects/__tests__/ObjectsBulkActions.test.js

@@ -20,19 +20,45 @@ import { ObjectsBulkActions } from "../ObjectsBulkActions"
 
 describe("ObjectsBulkActions", () => {
   it("should render without crashing", () => {
-    shallow(<ObjectsBulkActions checkedObjectsCount={0} />)
+    shallow(<ObjectsBulkActions checkedObjects={[]} />)
   })
 
   it("should show actions when checkObjectsCount is more than 0", () => {
-    const wrapper = shallow(<ObjectsBulkActions checkedObjectsCount={1} />)
+    const wrapper = shallow(<ObjectsBulkActions checkedObjects={["test"]} />)
     expect(wrapper.hasClass("list-actions-toggled")).toBeTruthy()
   })
 
-  it("should call downloadChecked when download button is clicked", () => {
+  it("should call downloadObject when single object is selected and download button is clicked", () => {
+    const downloadObject = jest.fn()
+    const clearChecked = jest.fn()
+    const wrapper = shallow(
+      <ObjectsBulkActions
+        checkedObjects={["test"]}
+        downloadObject={downloadObject}
+        clearChecked={clearChecked}
+      />
+    )
+    wrapper.find("#download-checked").simulate("click")
+    expect(downloadObject).toHaveBeenCalled()
+  })
+
+  it("should call downloadChecked when a folder is selected and download button is clicked", () => {
     const downloadChecked = jest.fn()
     const wrapper = shallow(
       <ObjectsBulkActions
-        checkedObjectsCount={1}
+        checkedObjects={["test/"]}
+        downloadChecked={downloadChecked}
+      />
+    )
+    wrapper.find("#download-checked").simulate("click")
+    expect(downloadChecked).toHaveBeenCalled()
+  })
+
+  it("should call downloadChecked when multiple objects are selected and download button is clicked", () => {
+    const downloadChecked = jest.fn()
+    const wrapper = shallow(
+      <ObjectsBulkActions
+        checkedObjects={["test1", "test2"]}
         downloadChecked={downloadChecked}
       />
     )
@@ -43,14 +69,14 @@ describe("ObjectsBulkActions", () => {
   it("should call clearChecked when close button is clicked", () => {
     const clearChecked = jest.fn()
     const wrapper = shallow(
-      <ObjectsBulkActions checkedObjectsCount={1} clearChecked={clearChecked} />
+      <ObjectsBulkActions checkedObjects={["test"]} clearChecked={clearChecked} />
     )
     wrapper.find("#close-bulk-actions").simulate("click")
     expect(clearChecked).toHaveBeenCalled()
   })
 
   it("shoud show DeleteObjectConfirmModal when delete-checked button is clicked", () => {
-    const wrapper = shallow(<ObjectsBulkActions checkedObjectsCount={1} />)
+    const wrapper = shallow(<ObjectsBulkActions checkedObjects={["test"]} />)
     wrapper.find("#delete-checked").simulate("click")
     wrapper.update()
     expect(wrapper.find("DeleteObjectConfirmModal").length).toBe(1)
@@ -60,7 +86,7 @@ describe("ObjectsBulkActions", () => {
     const deleteChecked = jest.fn()
     const wrapper = shallow(
       <ObjectsBulkActions
-        checkedObjectsCount={1}
+        checkedObjects={["test"]}
         deleteChecked={deleteChecked}
       />
     )

browser/app/less/inc/form.less

@@ -87,6 +87,7 @@ select.form-control {
     z-index: 1;
     border-bottom: 1px solid #eee;
     color: #32393F;
+    box-shadow: none;
     font-size: 13px;
 
 

browser/app/less/inc/header.less

@@ -44,9 +44,9 @@
 
 
 /*--------------------------
-    Disk usage
+    Disk used
 ----------------------------*/
-.feh-usage {
+.feh-used {
     margin-top: 12px;
     max-width: 285px;
 

buildscripts/checkdeps.sh

@@ -21,7 +21,7 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GO_VERSION="1.9.4"
+    GO_VERSION="1.10.1"
     OSX_VERSION="10.8"
     KNAME=$(uname -s)
     ARCH=$(uname -m)
@@ -34,7 +34,7 @@ _init() {
 
 ## FIXME:
 ## In OSX, 'readlink -f' option does not exist, hence
-## we have our own readlink -f behaviour here.
+## we have our own readlink -f behavior here.
 ## Once OSX has the option, below function is good enough.
 ## 
 ## readlink() {

buildscripts/gen-ldflags.go

@@ -33,6 +33,7 @@ func genLDFlags(version string) string {
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
 	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
+	ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT")
 	return ldflagsStr
 }
 

cmd/acl-handlers.go

@@ -0,0 +1,145 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/policy"
+)
+
+// Data types used for returning dummy access control
+// policy XML, these variables shouldn't be used elsewhere
+// they are only defined to be used in this file alone.
+type grantee struct {
+	XMLNS       string `xml:"xmlns:xsi,attr"`
+	XMLXSI      string `xml:"xsi:type,attr"`
+	Type        string `xml:"Type"`
+	ID          string `xml:"ID,omitempty"`
+	DisplayName string `xml:"DisplayName,omitempty"`
+	URI         string `xml:"URI,omitempty"`
+}
+
+type grant struct {
+	Grantee    grantee `xml:"Grantee"`
+	Permission string  `xml:"Permission"`
+}
+
+type accessControlPolicy struct {
+	XMLName           xml.Name `xml:"AccessControlPolicy"`
+	Owner             Owner    `xml:"Owner"`
+	AccessControlList struct {
+		Grants []grant `xml:"Grant"`
+	} `xml:"AccessControlList"`
+}
+
+// GetBucketACLHandler - GET Bucket ACL
+// -----------------
+// This operation uses the ACL
+// subresource to return the ACL of a specified bucket.
+func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, "GetBucketACL")
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Allow getBucketACL if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
+	// Before proceeding validate if bucket exists.
+	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	acl := &accessControlPolicy{}
+	acl.AccessControlList.Grants = append(acl.AccessControlList.Grants, grant{
+		Grantee: grantee{
+			XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
+			XMLXSI: "CanonicalUser",
+			Type:   "CanonicalUser",
+		},
+		Permission: "FULL_CONTROL",
+	})
+	if err := xml.NewEncoder(w).Encode(acl); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// GetObjectACLHandler - GET Object ACL
+// -----------------
+// This operation uses the ACL
+// subresource to return the ACL of a specified object.
+func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, "GetObjectACL")
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Allow getObjectACL if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
+	// Before proceeding validate if object exists.
+	_, err := objAPI.GetObjectInfo(ctx, bucket, object)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	acl := &accessControlPolicy{}
+	acl.AccessControlList.Grants = append(acl.AccessControlList.Grants, grant{
+		Grantee: grantee{
+			XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
+			XMLXSI: "CanonicalUser",
+			Type:   "CanonicalUser",
+		},
+		Permission: "FULL_CONTROL",
+	})
+	if err := xml.NewEncoder(w).Encode(acl); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}

cmd/admin-handlers.go

@@ -18,7 +18,9 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -27,9 +29,11 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/quick"
 )
 
 const (
@@ -62,16 +66,16 @@ var (
 // -----------
 // Returns Administration API version
 func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
-		writeErrorResponse(w, adminAPIErr, r.URL)
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
 	}
 
 	jsonBytes, err := json.Marshal(adminAPIVersionInfo)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to marshal Admin API Version to JSON.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -82,7 +86,7 @@ func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request)
 // ----------
 // Returns server version and uptime.
 func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -99,7 +103,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 	uptime, err := getPeerUptimes(globalAdminPeers)
 	if err != nil {
 		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
-		errorIf(err, "Possibly failed to get uptime from majority of servers.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -113,7 +117,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 	jsonBytes, err := json.Marshal(serverStatus)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to marshal storage info into json.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 	// Reply with storage information (across nodes in a
@@ -127,7 +131,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 // Restarts/Stops minio server gracefully. In a distributed setup,
 // restarts all the servers in the cluster.
 func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -136,7 +140,7 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 	var sa madmin.ServiceAction
 	err := json.NewDecoder(r.Body).Decode(&sa)
 	if err != nil {
-		errorIf(err, "Error parsing body JSON")
+		logger.LogIf(context.Background(), err)
 		writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
 		return
 	}
@@ -149,7 +153,7 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 		serviceSig = serviceStop
 	default:
 		writeErrorResponseJSON(w, ErrMalformedPOSTRequest, r.URL)
-		errorIf(err, "Invalid service action received")
+		logger.LogIf(context.Background(), errors.New("Invalid service action received"))
 		return
 	}
 
@@ -219,7 +223,9 @@ type ServerInfo struct {
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	// Authenticate request
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+
+	// Setting the region as empty so as the mc server info command is irrespective to the region.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -241,9 +247,11 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 			// Initialize server info at index
 			reply[idx] = ServerInfo{Addr: peer.addr}
 
-			serverInfoData, err := peer.cmdRunner.ServerInfoData()
+			serverInfoData, err := peer.cmdRunner.ServerInfo()
 			if err != nil {
-				errorIf(err, "Unable to get server info from %s.", peer.addr)
+				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peer.addr)
+				ctx := logger.SetReqInfo(context.Background(), reqInfo)
+				logger.LogIf(ctx, err)
 				reply[idx].Error = err.Error()
 				return
 			}
@@ -258,7 +266,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	jsonBytes, err := json.Marshal(reply)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to marshal storage info into json.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -292,7 +300,7 @@ func validateLockQueryParams(vars url.Values) (string, string, time.Duration,
 	}
 	duration, err := time.ParseDuration(olderThanStr)
 	if err != nil {
-		errorIf(err, "Failed to parse duration passed as query value.")
+		logger.LogIf(context.Background(), err)
 		return "", "", time.Duration(0), ErrInvalidDuration
 	}
 
@@ -306,7 +314,7 @@ func validateLockQueryParams(vars url.Values) (string, string, time.Duration,
 // Lists locks held on a given bucket, prefix and duration it was held for.
 func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Request) {
 
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -325,7 +333,7 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques
 		duration)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to fetch lock information from remote nodes.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -333,7 +341,7 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques
 	jsonBytes, err := json.Marshal(volLocks)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to marshal lock information into json.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -350,7 +358,14 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "ClearLocks")
 
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	// Get object layer instance.
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -369,7 +384,7 @@ func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Reque
 		duration)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to fetch lock information from remote nodes.")
+		logger.LogIf(ctx, err)
 		return
 	}
 
@@ -377,11 +392,11 @@ func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Reque
 	jsonBytes, err := json.Marshal(volLocks)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to marshal lock information into json.")
+		logger.LogIf(ctx, err)
 		return
 	}
 
-	newObjectLayerFn().ClearLocks(ctx, volLocks)
+	objLayer.ClearLocks(ctx, volLocks)
 
 	// Reply with list of locks cleared, as json.
 	writeSuccessResponseJSON(w, jsonBytes)
@@ -425,7 +440,7 @@ func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
 	if clientToken == "" {
 		jerr := json.NewDecoder(r.Body).Decode(&hs)
 		if jerr != nil {
-			errorIf(jerr, "Error parsing body JSON")
+			logger.LogIf(context.Background(), jerr)
 			err = ErrRequestBodyParse
 			return
 		}
@@ -459,7 +474,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -487,6 +502,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	// after 10s unless a response item comes in
 	keepConnLive := func(w http.ResponseWriter, respCh chan healResp) {
 		ticker := time.NewTicker(time.Second * 10)
+		defer ticker.Stop()
 		started := false
 	forLoop:
 		for {
@@ -515,7 +531,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 				break forLoop
 			}
 		}
-		ticker.Stop()
 	}
 
 	// find number of disks in the setup
@@ -555,20 +570,13 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 // GetConfigHandler - GET /minio/admin/v1/config
 // Get config.json of this minio setup.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
-
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
 	}
 
-	// check if objectLayer is initialized, if not return.
-	if newObjectLayerFn() == nil {
-		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
-		return
-	}
-
 	// Take a read lock on minio/config.json. NB minio is a
 	// reserved bucket name and wouldn't conflict with normal
 	// object operations.
@@ -583,7 +591,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	// occurring on a quorum of the servers is returned.
 	configBytes, err := getPeerConfig(globalAdminPeers)
 	if err != nil {
-		errorIf(err, "Failed to get config from peers")
+		logger.LogIf(context.Background(), err)
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		return
 	}
@@ -655,6 +663,7 @@ func writeSetConfigResponse(w http.ResponseWriter, peers adminPeers,
 // SetConfigHandler - PUT /minio/admin/v1/config
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 
+	ctx := context.Background()
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -662,8 +671,14 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -674,12 +689,12 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	n, err := io.ReadFull(r.Body, configBuf)
 	if err == nil {
 		// More than maxConfigSize bytes were available
-		writeErrorResponse(w, ErrAdminConfigTooLarge, r.URL)
+		writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
 		return
 	}
 	if err != io.ErrUnexpectedEOF {
-		errorIf(err, "Failed to read config from request body.")
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
@@ -687,17 +702,17 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	// Validate JSON provided in the request body: check the
 	// client has not sent JSON objects with duplicate keys.
-	if err = checkDupJSONKeys(string(configBytes)); err != nil {
-		errorIf(err, "config contains duplicate JSON entries.")
-		writeErrorResponse(w, ErrAdminConfigBadJSON, r.URL)
+	if err = quick.CheckDuplicateKeys(string(configBytes)); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
 		return
 	}
 
 	var config serverConfig
 	err = json.Unmarshal(configBytes, &config)
 	if err != nil {
-		errorIf(err, "Failed to unmarshal JSON configuration", err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
@@ -718,7 +733,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	errs := writeTmpConfigPeers(globalAdminPeers, tmpFileName, configBytes)
 
 	// Check if the operation succeeded in quorum or more nodes.
-	rErr := reduceWriteQuorumErrs(errs, nil, len(globalAdminPeers)/2+1)
+	rErr := reduceWriteQuorumErrs(ctx, errs, nil, len(globalAdminPeers)/2+1)
 	if rErr != nil {
 		writeSetConfigResponse(w, globalAdminPeers, errs, false, r.URL)
 		return
@@ -736,7 +751,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	// Rename the temporary config file to config.json
 	errs = commitConfigPeers(globalAdminPeers, tmpFileName)
-	rErr = reduceWriteQuorumErrs(errs, nil, len(globalAdminPeers)/2+1)
+	rErr = reduceWriteQuorumErrs(ctx, errs, nil, len(globalAdminPeers)/2+1)
 	if rErr != nil {
 		writeSetConfigResponse(w, globalAdminPeers, errs, false, r.URL)
 		return
@@ -760,16 +775,16 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 	r *http.Request) {
 
 	// Authenticate request
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
-		writeErrorResponse(w, adminAPIErr, r.URL)
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
 	}
 
 	// Avoid setting new credentials when they are already passed
-	// by the environment.
-	if globalIsEnvCreds {
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+	// by the environment. Deny if WORM is enabled.
+	if globalIsEnvCreds || globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
 		return
 	}
 
@@ -777,14 +792,14 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 	var req madmin.SetCredsReq
 	err := json.NewDecoder(r.Body).Decode(&req)
 	if err != nil {
-		errorIf(err, "Error parsing body JSON")
+		logger.LogIf(context.Background(), err)
 		writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
 		return
 	}
 
 	creds, err := auth.CreateCredentials(req.AccessKey, req.SecretKey)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
@@ -801,19 +816,20 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 	globalServerConfigMu.Lock()
 	defer globalServerConfigMu.Unlock()
 
-	// Notify all other Minio peers to update credentials
-	updateErrs := updateCredsOnPeers(creds)
-	for peer, err := range updateErrs {
-		errorIf(err, "Unable to update credentials on peer %s.", peer)
-	}
-
 	// Update local credentials in memory.
 	globalServerConfig.SetCredential(creds)
-	if err = globalServerConfig.Save(); err != nil {
-		writeErrorResponse(w, ErrInternalError, r.URL)
+	if err = globalServerConfig.Save(getConfigFile()); err != nil {
+		writeErrorResponseJSON(w, ErrInternalError, r.URL)
 		return
 	}
 
+	// Notify all other Minio peers to update credentials
+	for host, err := range globalNotificationSys.SetCredentials(creds) {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", host.String())
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+	}
+
 	// At this stage, the operation is successful, return 200 OK
 	w.WriteHeader(http.StatusOK)
 }

cmd/admin-handlers_test.go

@@ -31,9 +31,8 @@ import (
 	"testing"
 	"time"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -141,7 +140,7 @@ type adminXLTestBed struct {
 	configPath string
 	xlDirs     []string
 	objLayer   ObjectLayer
-	mux        *router.Router
+	router     *mux.Router
 }
 
 // prepareAdminXLTestBed - helper function that setups a single-node
@@ -182,15 +181,18 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 		return nil, err
 	}
 
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
+
 	// Setup admin mgmt REST API handlers.
-	adminRouter := router.NewRouter()
+	adminRouter := mux.NewRouter()
 	registerAdminRouter(adminRouter)
 
 	return &adminXLTestBed{
 		configPath: rootPath,
 		xlDirs:     xlDirs,
 		objLayer:   objLayer,
-		mux:        adminRouter,
+		router:     adminRouter,
 	}, nil
 }
 
@@ -264,12 +266,13 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) {
 		return nil, nil, err
 	}
 	endpoints := mustGetNewEndpointList(xlDirs...)
-	format, err := waitForFormatXL(true, endpoints, 1, 16)
+	format, err := waitForFormatXL(context.Background(), true, endpoints, 1, 16)
 	if err != nil {
 		removeRoots(xlDirs)
 		return nil, nil, err
 	}
 
+	globalPolicySys = NewPolicySys()
 	objLayer, err := newXLSets(endpoints, format, 1, 16)
 	if err != nil {
 		return nil, nil, err
@@ -300,7 +303,7 @@ func TestAdminVersionHandler(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if http.StatusOK != rec.Code {
 		t.Errorf("Unexpected status code - got %d but expected %d",
 			rec.Code, http.StatusOK)
@@ -441,7 +444,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
-	// Setting up a go routine to simulate ServerMux's
+	// Setting up a go routine to simulate ServerRouter's
 	// handleServiceSignals for stop and restart commands.
 	if cmd == restartCmd {
 		go testServiceSignalReceiver(cmd, t)
@@ -460,7 +463,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 
 	if cmd == statusCmd {
 		expectedInfo := madmin.ServiceStatus{
@@ -544,7 +547,7 @@ func TestServiceSetCreds(t *testing.T) {
 		rec := httptest.NewRecorder()
 
 		// Execute request
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 
 		// Check if the http code response is expected
 		if rec.Code != testCase.ExpectedStatusCode {
@@ -637,7 +640,7 @@ func TestListLocksHandler(t *testing.T) {
 			t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if test.expectedStatus != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
 		}
@@ -704,7 +707,7 @@ func TestClearLocksHandler(t *testing.T) {
 			t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if test.expectedStatus != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
 		}
@@ -762,13 +765,13 @@ func buildAdminRequest(queryVal url.Values, method, path string,
 		"/minio/admin/v1"+path+"?"+queryVal.Encode(),
 		contentLength, bodySeeker)
 	if err != nil {
-		return nil, errors.Trace(err)
+		return nil, err
 	}
 
 	cred := globalServerConfig.GetCredential()
 	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 	if err != nil {
-		return nil, errors.Trace(err)
+		return nil, err
 	}
 
 	return req, nil
@@ -796,7 +799,7 @@ func TestGetConfigHandler(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
@@ -830,7 +833,7 @@ func TestSetConfigHandler(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
@@ -856,7 +859,7 @@ func TestSetConfigHandler(t *testing.T) {
 		}
 
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		respBody := string(rec.Body.Bytes())
 		if rec.Code != http.StatusBadRequest ||
 			!strings.Contains(respBody, "Configuration data provided exceeds the allowed maximum of") {
@@ -875,7 +878,7 @@ func TestSetConfigHandler(t *testing.T) {
 		}
 
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		respBody := string(rec.Body.Bytes())
 		if rec.Code != http.StatusBadRequest ||
 			!strings.Contains(respBody, "JSON configuration provided has objects with duplicate keys") {
@@ -905,7 +908,7 @@ func TestAdminServerInfo(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
@@ -927,9 +930,6 @@ func TestAdminServerInfo(t *testing.T) {
 		if serverInfo.Error != "" {
 			t.Errorf("Unexpected error = %v\n", serverInfo.Error)
 		}
-		if serverInfo.Data.StorageInfo.Free == 0 {
-			t.Error("Expected StorageInfo.Free to be non empty")
-		}
 		if serverInfo.Data.Properties.Region != globalMinioDefaultRegion {
 			t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, serverInfo.Data.Properties.Region)
 		}
@@ -1106,7 +1106,7 @@ func collectHealResults(t *testing.T, adminTestBed *adminXLTestBed, bucket,
 		}
 		req := mkHealStatusReq(t, bucket, prefix, clientToken)
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if http.StatusOK != rec.Code {
 			t.Errorf("Unexpected status code - got %d but expected %d",
 				rec.Code, http.StatusOK)
@@ -1152,7 +1152,7 @@ func TestHealStartNStatusHandler(t *testing.T) {
 	{
 		req := mkHealStartReq(t, bucketName, objName, healOpts)
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if http.StatusOK != rec.Code {
 			t.Errorf("Unexpected status code - got %d but expected %d",
 				rec.Code, http.StatusOK)
@@ -1172,7 +1172,7 @@ func TestHealStartNStatusHandler(t *testing.T) {
 		// test with an invalid client token
 		req := mkHealStatusReq(t, bucketName, objName, hss.ClientToken+hss.ClientToken)
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if rec.Code != http.StatusBadRequest {
 			t.Errorf("Unexpected status code")
 		}

cmd/admin-heal-ops.go

@@ -57,7 +57,7 @@ const (
 var (
 	errHealIdleTimeout      = fmt.Errorf("healing results were not consumed for too long")
 	errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
-	errHealStopSignalled    = fmt.Errorf("heal stop signalled")
+	errHealStopSignalled    = fmt.Errorf("heal stop signaled")
 
 	errFnHealFromAPIErr = func(err error) error {
 		errCode := toAPIErrorCode(err)
@@ -222,7 +222,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 		StartTime:     h.startTime,
 	})
 	if err != nil {
-		errorIf(err, "Failed to marshal heal result into json.")
+		logger.LogIf(context.Background(), err)
 		return nil, ErrInternalError, ""
 	}
 	return b, ErrNone, ""
@@ -270,7 +270,7 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 
 	jbytes, err := json.Marshal(h.currentStatus)
 	if err != nil {
-		errorIf(err, "Failed to marshal heal result into json.")
+		logger.LogIf(context.Background(), err)
 		return nil, ErrInternalError
 	}
 
@@ -301,7 +301,7 @@ type healSequence struct {
 	// current accumulated status of the heal sequence
 	currentStatus healSequenceStatus
 
-	// channel signalled by background routine when traversal has
+	// channel signaled by background routine when traversal has
 	// completed
 	traverseAndHealDoneCh chan error
 
@@ -321,7 +321,9 @@ type healSequence struct {
 func newHealSequence(bucket, objPrefix, clientAddr string,
 	numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence {
 
-	ctx := logger.SetContext(context.Background(), &logger.ReqInfo{clientAddr, "", "", "Heal", bucket, objPrefix, nil})
+	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
+	reqInfo.AppendTags("prefix", objPrefix)
+	ctx := logger.SetReqInfo(context.Background(), reqInfo)
 
 	return &healSequence{
 		bucket:        bucket,
@@ -439,7 +441,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 	h.currentStatus.updateLock.Unlock()
 
 	// This is a "safe" point for the heal sequence to quit if
-	// signalled externally.
+	// signaled externally.
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
@@ -537,11 +539,17 @@ func (h *healSequence) healDiskFormat() error {
 	}
 
 	res, err := objectAPI.HealFormat(h.ctx, h.settings.DryRun)
-	if err != nil {
+	// return any error, ignore error returned when disks have
+	// already healed.
+	if err != nil && err != errNoHealRequired {
 		return errFnHealFromAPIErr(err)
 	}
 
-	peersReInitFormat(globalAdminPeers, h.settings.DryRun)
+	// Healing succeeded notify the peers to reload format and re-initialize disks.
+	// We will not notify peers only if healing succeeded.
+	if err == nil {
+		peersReInitFormat(globalAdminPeers, h.settings.DryRun)
+	}
 
 	// Push format heal result
 	return h.pushHealResultItem(res)

cmd/admin-router.go

@@ -19,7 +19,7 @@ package cmd
 import (
 	"net/http"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 )
 
 const (
@@ -31,48 +31,48 @@ type adminAPIHandlers struct {
 }
 
 // registerAdminRouter - Add handler functions for each service REST API routes.
-func registerAdminRouter(mux *router.Router) {
+func registerAdminRouter(router *mux.Router) {
 
 	adminAPI := adminAPIHandlers{}
 	// Admin router
-	adminRouter := mux.NewRoute().PathPrefix(adminAPIPathPrefix).Subrouter()
+	adminRouter := router.PathPrefix(adminAPIPathPrefix).Subrouter()
 
 	// Version handler
-	adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(adminAPI.VersionHandler)
+	adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(httpTraceAll(adminAPI.VersionHandler))
 
 	adminV1Router := adminRouter.PathPrefix("/v1").Subrouter()
 
 	/// Service operations
 
 	// Service status
-	adminV1Router.Methods(http.MethodGet).Path("/service").HandlerFunc(adminAPI.ServiceStatusHandler)
+	adminV1Router.Methods(http.MethodGet).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceStatusHandler))
 
 	// Service restart and stop - TODO
-	adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(adminAPI.ServiceStopNRestartHandler)
+	adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceStopNRestartHandler))
 
 	// Info operations
-	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(adminAPI.ServerInfoHandler)
+	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
 
 	/// Lock operations
 
 	// List Locks
-	adminV1Router.Methods(http.MethodGet).Path("/locks").HandlerFunc(adminAPI.ListLocksHandler)
+	adminV1Router.Methods(http.MethodGet).Path("/locks").HandlerFunc(httpTraceAll(adminAPI.ListLocksHandler))
 	// Clear locks
-	adminV1Router.Methods(http.MethodDelete).Path("/locks").HandlerFunc(adminAPI.ClearLocksHandler)
+	adminV1Router.Methods(http.MethodDelete).Path("/locks").HandlerFunc(httpTraceAll(adminAPI.ClearLocksHandler))
 
 	/// Heal operations
 
 	// Heal processing endpoint.
-	adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(adminAPI.HealHandler)
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(adminAPI.HealHandler)
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(adminAPI.HealHandler)
+	adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
 	/// Config operations
 
 	// Update credentials
-	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(adminAPI.UpdateCredentialsHandler)
+	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceAll(adminAPI.UpdateCredentialsHandler))
 	// Get config
-	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(adminAPI.GetConfigHandler)
+	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceAll(adminAPI.GetConfigHandler))
 	// Set config
-	adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(adminAPI.SetConfigHandler)
+	adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceAll(adminAPI.SetConfigHandler))
 }

cmd/admin-rpc-client.go

@@ -18,39 +18,128 @@ package cmd
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"net"
-	"os"
-	"path"
-	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
-	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
-const (
-	// Admin service names
-	signalServiceRPC  = "Admin.SignalService"
-	reInitFormatRPC   = "Admin.ReInitFormat"
-	listLocksRPC      = "Admin.ListLocks"
-	serverInfoDataRPC = "Admin.ServerInfoData"
-	getConfigRPC      = "Admin.GetConfig"
-	writeTmpConfigRPC = "Admin.WriteTmpConfig"
-	commitConfigRPC   = "Admin.CommitConfig"
-)
+var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
 
-// localAdminClient - represents admin operation to be executed locally.
-type localAdminClient struct {
+// AdminRPCClient - admin RPC client talks to admin RPC server.
+type AdminRPCClient struct {
+	*RPCClient
 }
 
-// remoteAdminClient - represents admin operation to be executed
-// remotely, via RPC.
-type remoteAdminClient struct {
-	*AuthRPCClient
+// SignalService - calls SignalService RPC.
+func (rpcClient *AdminRPCClient) SignalService(signal serviceSignal) (err error) {
+	args := SignalServiceArgs{Sig: signal}
+	reply := VoidReply{}
+
+	return rpcClient.Call(adminServiceName+".SignalService", &args, &reply)
+}
+
+// ReInitFormat - re-initialize disk format, remotely.
+func (rpcClient *AdminRPCClient) ReInitFormat(dryRun bool) error {
+	args := ReInitFormatArgs{DryRun: dryRun}
+	reply := VoidReply{}
+
+	return rpcClient.Call(adminServiceName+".ReInitFormat", &args, &reply)
+}
+
+// ListLocks - Sends list locks command to remote server via RPC.
+func (rpcClient *AdminRPCClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
+	args := ListLocksQuery{
+		Bucket:   bucket,
+		Prefix:   prefix,
+		Duration: duration,
+	}
+	var reply []VolumeLockInfo
+
+	err := rpcClient.Call(adminServiceName+".ListLocks", &args, &reply)
+	return reply, err
+}
+
+// ServerInfo - returns the server info of the server to which the RPC call is made.
+func (rpcClient *AdminRPCClient) ServerInfo() (sid ServerInfoData, err error) {
+	err = rpcClient.Call(adminServiceName+".ServerInfo", &AuthArgs{}, &sid)
+	return sid, err
+}
+
+// GetConfig - returns config.json of the remote server.
+func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
+	args := AuthArgs{}
+	var reply []byte
+
+	err := rpcClient.Call(adminServiceName+".GetConfig", &args, &reply)
+	return reply, err
+}
+
+// WriteTmpConfig - writes config file content to a temporary file on a remote node.
+func (rpcClient *AdminRPCClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
+	args := WriteConfigArgs{
+		TmpFileName: tmpFileName,
+		Buf:         configBytes,
+	}
+	reply := VoidReply{}
+
+	err := rpcClient.Call(adminServiceName+".WriteTmpConfig", &args, &reply)
+	logger.LogIf(context.Background(), err)
+	return err
+}
+
+// CommitConfig - Move the new config in tmpFileName onto config.json on a remote node.
+func (rpcClient *AdminRPCClient) CommitConfig(tmpFileName string) error {
+	args := CommitConfigArgs{FileName: tmpFileName}
+	reply := VoidReply{}
+
+	err := rpcClient.Call(adminServiceName+".CommitConfig", &args, &reply)
+	logger.LogIf(context.Background(), err)
+	return err
+}
+
+// NewAdminRPCClient - returns new admin RPC client.
+func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
+	scheme := "http"
+	if globalIsSSL {
+		scheme = "https"
+	}
+
+	serviceURL := &xnet.URL{
+		Scheme: scheme,
+		Host:   host.String(),
+		Path:   adminServicePath,
+	}
+
+	var tlsConfig *tls.Config
+	if globalIsSSL {
+		tlsConfig = &tls.Config{
+			ServerName: host.Name,
+			RootCAs:    globalRootCAs,
+		}
+	}
+
+	rpcClient, err := NewRPCClient(
+		RPCClientArgs{
+			NewAuthTokenFunc: newAuthToken,
+			RPCVersion:       globalRPCAPIVersion,
+			ServiceName:      adminServiceName,
+			ServiceURL:       serviceURL,
+			TLSConfig:        tlsConfig,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &AdminRPCClient{rpcClient}, nil
 }
 
 // adminCmdRunner - abstracts local and remote execution of admin
@@ -59,182 +148,12 @@ type adminCmdRunner interface {
 	SignalService(s serviceSignal) error
 	ReInitFormat(dryRun bool) error
 	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
-	ServerInfoData() (ServerInfoData, error)
+	ServerInfo() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
 	WriteTmpConfig(tmpFileName string, configBytes []byte) error
 	CommitConfig(tmpFileName string) error
 }
 
-var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
-
-// SignalService - sends a restart or stop signal to the local server
-func (lc localAdminClient) SignalService(s serviceSignal) error {
-	switch s {
-	case serviceRestart, serviceStop:
-		globalServiceSignalCh <- s
-	default:
-		return errUnsupportedSignal
-	}
-	return nil
-}
-
-// ReInitFormat - re-initialize disk format.
-func (lc localAdminClient) ReInitFormat(dryRun bool) error {
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	_, err := objectAPI.HealFormat(context.Background(), dryRun)
-	return err
-}
-
-// ListLocks - Fetches lock information from local lock instrumentation.
-func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	return listLocksInfo(bucket, prefix, duration), nil
-}
-
-func (rc remoteAdminClient) SignalService(s serviceSignal) (err error) {
-	switch s {
-	case serviceRestart, serviceStop:
-		reply := AuthRPCReply{}
-		err = rc.Call(signalServiceRPC, &SignalServiceArgs{Sig: s},
-			&reply)
-	default:
-		err = errUnsupportedSignal
-	}
-	return err
-}
-
-// ReInitFormat - re-initialize disk format, remotely.
-func (rc remoteAdminClient) ReInitFormat(dryRun bool) error {
-	reply := AuthRPCReply{}
-	return rc.Call(reInitFormatRPC, &ReInitFormatArgs{
-		DryRun: dryRun,
-	}, &reply)
-}
-
-// ListLocks - Sends list locks command to remote server via RPC.
-func (rc remoteAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	listArgs := ListLocksQuery{
-		Bucket:   bucket,
-		Prefix:   prefix,
-		Duration: duration,
-	}
-	var reply ListLocksReply
-	if err := rc.Call(listLocksRPC, &listArgs, &reply); err != nil {
-		return nil, err
-	}
-	return reply.VolLocks, nil
-}
-
-// ServerInfoData - Returns the server info of this server.
-func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	if globalBootTime.IsZero() {
-		return sid, errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return sid, errServerNotInitialized
-	}
-	storage := objLayer.StorageInfo(context.Background())
-
-	return ServerInfoData{
-		StorageInfo: storage,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			SQSARN:   globalNotificationSys.GetARNList(),
-			Region:   globalServerConfig.GetRegion(),
-		},
-	}, nil
-}
-
-// ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	args := AuthRPCArgs{}
-	reply := ServerInfoDataReply{}
-	err := rc.Call(serverInfoDataRPC, &args, &reply)
-	if err != nil {
-		return sid, err
-	}
-
-	return reply.ServerInfoData, nil
-}
-
-// GetConfig - returns config.json of the local server.
-func (lc localAdminClient) GetConfig() ([]byte, error) {
-	if globalServerConfig == nil {
-		return nil, fmt.Errorf("config not present")
-	}
-
-	return json.Marshal(globalServerConfig)
-}
-
-// GetConfig - returns config.json of the remote server.
-func (rc remoteAdminClient) GetConfig() ([]byte, error) {
-	args := AuthRPCArgs{}
-	reply := ConfigReply{}
-	if err := rc.Call(getConfigRPC, &args, &reply); err != nil {
-		return nil, err
-	}
-	return reply.Config, nil
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// the local server.
-func (lc localAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	return writeTmpConfigCommon(tmpFileName, configBytes)
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// a remote node.
-func (rc remoteAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	wArgs := WriteConfigArgs{
-		TmpFileName: tmpFileName,
-		Buf:         configBytes,
-	}
-
-	err := rc.Call(writeTmpConfigRPC, &wArgs, &WriteConfigReply{})
-	if err != nil {
-		errorIf(err, "Failed to write temporary config file.")
-		return err
-	}
-
-	return nil
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a local node.
-func (lc localAdminClient) CommitConfig(tmpFileName string) error {
-	configFile := getConfigFile()
-	tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName)
-
-	err := os.Rename(tmpConfigFile, configFile)
-	errorIf(err, fmt.Sprintf("Failed to rename %s to %s", tmpConfigFile, configFile))
-	return err
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a remote node.
-func (rc remoteAdminClient) CommitConfig(tmpFileName string) error {
-	cArgs := CommitConfigArgs{
-		FileName: tmpFileName,
-	}
-	cReply := CommitConfigReply{}
-	err := rc.Call(commitConfigRPC, &cArgs, &cReply)
-	if err != nil {
-		errorIf(err, "Failed to rename config file.")
-		return err
-	}
-
-	return nil
-}
-
 // adminPeer - represents an entity that implements admin API RPCs.
 type adminPeer struct {
 	addr      string
@@ -247,34 +166,25 @@ type adminPeers []adminPeer
 
 // makeAdminPeers - helper function to construct a collection of adminPeer.
 func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
-	thisPeer := globalMinioAddr
-	if globalMinioHost == "" {
-		thisPeer = net.JoinHostPort("localhost", globalMinioPort)
+	localAddr := GetLocalPeer(endpoints)
+	if strings.HasPrefix(localAddr, "127.0.0.1:") {
+		// Use first IPv4 instead of loopback address.
+		localAddr = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
 	}
 	adminPeerList = append(adminPeerList, adminPeer{
-		thisPeer,
-		localAdminClient{},
-		true,
+		addr:      localAddr,
+		cmdRunner: localAdminClient{},
+		isLocal:   true,
 	})
 
-	hostSet := set.CreateStringSet(globalMinioAddr)
-	cred := globalServerConfig.GetCredential()
-	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
-	for _, host := range GetRemotePeers(endpoints) {
-		if hostSet.Contains(host) {
-			continue
-		}
-		hostSet.Add(host)
+	for _, hostStr := range GetRemotePeers(endpoints) {
+		host, err := xnet.ParseHost(hostStr)
+		logger.FatalIf(err, "Unable to parse Admin RPC Host", context.Background())
+		rpcClient, err := NewAdminRPCClient(host)
+		logger.FatalIf(err, "Unable to initialize Admin RPC Client", context.Background())
 		adminPeerList = append(adminPeerList, adminPeer{
-			addr: host,
-			cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
-				accessKey:       cred.AccessKey,
-				secretKey:       cred.SecretKey,
-				serverAddr:      host,
-				serviceEndpoint: serviceEndpoint,
-				secureConn:      globalIsSSL,
-				serviceName:     "Admin",
-			})},
+			addr:      hostStr,
+			cmdRunner: rpcClient,
 		})
 	}
 
@@ -420,7 +330,7 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
 		wg.Add(1)
 		go func(idx int, peer adminPeer) {
 			defer wg.Done()
-			serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
+			serverInfoData, rpcErr := peer.cmdRunner.ServerInfo()
 			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
 		}(i, peer)
 	}
@@ -436,7 +346,7 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
 	latestUptime := time.Duration(0)
 	for _, uptime := range uptimes {
 		if uptime.err != nil {
-			errorIf(uptime.err, "Unable to fetch uptime")
+			logger.LogIf(context.Background(), uptime.err)
 			continue
 		}
 
@@ -489,15 +399,17 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
 		// Unmarshal the received config files.
 		err := json.Unmarshal(configBytes, &serverConfigs[i])
 		if err != nil {
-			errorIf(err, "Failed to unmarshal serverConfig from ", peers[i].addr)
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peers[i].addr)
+			ctx := logger.SetReqInfo(context.Background(), reqInfo)
+			logger.LogIf(ctx, err)
 			return nil, err
 		}
 	}
 
 	configJSON, err := getValidServerConfig(serverConfigs, errs)
 	if err != nil {
-		errorIf(err, "Unable to find a valid server config")
-		return nil, errors.Trace(err)
+		logger.LogIf(context.Background(), err)
+		return nil, err
 	}
 
 	// Return the config.json that was present quorum or more

cmd/admin-rpc-client.go.orig

@@ -0,0 +1,650 @@
+/*
+ * Minio Cloud Storage, (C) 2014, 2015, 2016, 2017, 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/cmd/logger"
+)
+
+const (
+	// Admin service names
+	signalServiceRPC  = "Admin.SignalService"
+	reInitFormatRPC   = "Admin.ReInitFormat"
+	listLocksRPC      = "Admin.ListLocks"
+	serverInfoDataRPC = "Admin.ServerInfoData"
+	getConfigRPC      = "Admin.GetConfig"
+	writeTmpConfigRPC = "Admin.WriteTmpConfig"
+	commitConfigRPC   = "Admin.CommitConfig"
+)
+
+// localAdminClient - represents admin operation to be executed locally.
+type localAdminClient struct {
+}
+
+// remoteAdminClient - represents admin operation to be executed
+// remotely, via RPC.
+type remoteAdminClient struct {
+	*AuthRPCClient
+}
+
+// adminCmdRunner - abstracts local and remote execution of admin
+// commands like service stop and service restart.
+type adminCmdRunner interface {
+	SignalService(s serviceSignal) error
+	ReInitFormat(dryRun bool) error
+	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
+	ServerInfoData() (ServerInfoData, error)
+	GetConfig() ([]byte, error)
+	WriteTmpConfig(tmpFileName string, configBytes []byte) error
+	CommitConfig(tmpFileName string) error
+}
+
+var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
+
+// SignalService - sends a restart or stop signal to the local server
+func (lc localAdminClient) SignalService(s serviceSignal) error {
+	switch s {
+	case serviceRestart, serviceStop:
+		globalServiceSignalCh <- s
+	default:
+		return errUnsupportedSignal
+	}
+	return nil
+}
+
+// ReInitFormat - re-initialize disk format.
+func (lc localAdminClient) ReInitFormat(dryRun bool) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+	return objectAPI.ReloadFormat(context.Background(), dryRun)
+}
+
+// ListLocks - Fetches lock information from local lock instrumentation.
+func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
+	// check if objectLayer is initialized, if not return.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return nil, errServerNotInitialized
+	}
+	return objectAPI.ListLocks(context.Background(), bucket, prefix, duration)
+}
+
+func (rc remoteAdminClient) SignalService(s serviceSignal) (err error) {
+	switch s {
+	case serviceRestart, serviceStop:
+		reply := AuthRPCReply{}
+		err = rc.Call(signalServiceRPC, &SignalServiceArgs{Sig: s},
+			&reply)
+	default:
+		err = errUnsupportedSignal
+	}
+	return err
+}
+
+// ReInitFormat - re-initialize disk format, remotely.
+func (rc remoteAdminClient) ReInitFormat(dryRun bool) error {
+	reply := AuthRPCReply{}
+	return rc.Call(reInitFormatRPC, &ReInitFormatArgs{
+		DryRun: dryRun,
+	}, &reply)
+}
+
+// ListLocks - Sends list locks command to remote server via RPC.
+func (rc remoteAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
+	listArgs := ListLocksQuery{
+		Bucket:   bucket,
+		Prefix:   prefix,
+		Duration: duration,
+	}
+	var reply ListLocksReply
+	if err := rc.Call(listLocksRPC, &listArgs, &reply); err != nil {
+		return nil, err
+	}
+	return reply.VolLocks, nil
+}
+
+// ServerInfoData - Returns the server info of this server.
+func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
+	if globalBootTime.IsZero() {
+		return sid, errServerNotInitialized
+	}
+
+	// Build storage info
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		return sid, errServerNotInitialized
+	}
+	storage := objLayer.StorageInfo(context.Background())
+
+	return ServerInfoData{
+		StorageInfo: storage,
+		ConnStats:   globalConnStats.toServerConnStats(),
+		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+		Properties: ServerProperties{
+			Uptime:   UTCNow().Sub(globalBootTime),
+			Version:  Version,
+			CommitID: CommitID,
+			SQSARN:   globalNotificationSys.GetARNList(),
+			Region:   globalServerConfig.GetRegion(),
+		},
+	}, nil
+}
+
+// ServerInfo - returns the server info of the server to which the RPC call is made.
+func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
+	args := AuthRPCArgs{}
+	reply := ServerInfoDataReply{}
+	err := rc.Call(serverInfoDataRPC, &args, &reply)
+	if err != nil {
+		return sid, err
+	}
+
+	return reply.ServerInfoData, nil
+}
+
+// GetConfig - returns config.json of the local server.
+func (lc localAdminClient) GetConfig() ([]byte, error) {
+	if globalServerConfig == nil {
+		return nil, fmt.Errorf("config not present")
+	}
+
+	return json.Marshal(globalServerConfig)
+}
+
+// GetConfig - returns config.json of the remote server.
+func (rc remoteAdminClient) GetConfig() ([]byte, error) {
+	args := AuthRPCArgs{}
+	reply := ConfigReply{}
+	if err := rc.Call(getConfigRPC, &args, &reply); err != nil {
+		return nil, err
+	}
+	return reply.Config, nil
+}
+
+// WriteTmpConfig - writes config file content to a temporary file on
+// the local server.
+func (lc localAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
+	return writeTmpConfigCommon(tmpFileName, configBytes)
+}
+
+// WriteTmpConfig - writes config file content to a temporary file on
+// a remote node.
+func (rc remoteAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
+	wArgs := WriteConfigArgs{
+		TmpFileName: tmpFileName,
+		Buf:         configBytes,
+	}
+
+	err := rc.Call(writeTmpConfigRPC, &wArgs, &WriteConfigReply{})
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return err
+	}
+
+	return nil
+}
+
+// CommitConfig - Move the new config in tmpFileName onto config.json
+// on a local node.
+func (lc localAdminClient) CommitConfig(tmpFileName string) error {
+	configFile := getConfigFile()
+	tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName)
+
+	err := os.Rename(tmpConfigFile, configFile)
+	reqInfo := (&logger.ReqInfo{}).AppendTags("tmpConfigFile", tmpConfigFile)
+	reqInfo.AppendTags("configFile", configFile)
+	ctx := logger.SetReqInfo(context.Background(), reqInfo)
+	logger.LogIf(ctx, err)
+	return err
+}
+
+// CommitConfig - Move the new config in tmpFileName onto config.json
+// on a remote node.
+func (rc remoteAdminClient) CommitConfig(tmpFileName string) error {
+	cArgs := CommitConfigArgs{
+		FileName: tmpFileName,
+	}
+	cReply := CommitConfigReply{}
+	err := rc.Call(commitConfigRPC, &cArgs, &cReply)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return err
+	}
+
+	return nil
+}
+
+// adminPeer - represents an entity that implements admin API RPCs.
+type adminPeer struct {
+	addr      string
+	cmdRunner adminCmdRunner
+	isLocal   bool
+}
+
+// type alias for a collection of adminPeer.
+type adminPeers []adminPeer
+
+// makeAdminPeers - helper function to construct a collection of adminPeer.
+func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
+	thisPeer := globalMinioAddr
+	if globalMinioHost == "" {
+		// When host is not explicitly provided simply
+		// use the first IPv4.
+		thisPeer = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
+	}
+	adminPeerList = append(adminPeerList, adminPeer{
+		thisPeer,
+		localAdminClient{},
+		true,
+	})
+
+	hostSet := set.CreateStringSet(globalMinioAddr)
+	cred := globalServerConfig.GetCredential()
+	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
+	for _, host := range GetRemotePeers(endpoints) {
+		if hostSet.Contains(host) {
+			continue
+		}
+		hostSet.Add(host)
+		adminPeerList = append(adminPeerList, adminPeer{
+			addr: host,
+			cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
+				accessKey:       cred.AccessKey,
+				secretKey:       cred.SecretKey,
+				serverAddr:      host,
+				serviceEndpoint: serviceEndpoint,
+				secureConn:      globalIsSSL,
+				serviceName:     "Admin",
+			})},
+		})
+	}
+
+	return adminPeerList
+}
+
+// peersReInitFormat - reinitialize remote object layers to new format.
+func peersReInitFormat(peers adminPeers, dryRun bool) error {
+	errs := make([]error, len(peers))
+
+	// Send ReInitFormat RPC call to all nodes.
+	// for local adminPeer this is a no-op.
+	wg := sync.WaitGroup{}
+	for i, peer := range peers {
+		wg.Add(1)
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+			if !peer.isLocal {
+				errs[idx] = peer.cmdRunner.ReInitFormat(dryRun)
+			}
+		}(i, peer)
+	}
+	wg.Wait()
+	return nil
+}
+
+// Initialize global adminPeer collection.
+func initGlobalAdminPeers(endpoints EndpointList) {
+	globalAdminPeers = makeAdminPeers(endpoints)
+}
+
+// invokeServiceCmd - Invoke Restart/Stop command.
+func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
+	switch cmd {
+	case serviceRestart, serviceStop:
+		err = cp.cmdRunner.SignalService(cmd)
+	}
+	return err
+}
+
+// sendServiceCmd - Invoke Restart command on remote peers
+// adminPeer followed by on the local peer.
+func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
+	// Send service command like stop or restart to all remote nodes and finally run on local node.
+	errs := make([]error, len(cps))
+	var wg sync.WaitGroup
+	remotePeers := cps[1:]
+	for i := range remotePeers {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			// we use idx+1 because remotePeers slice is 1 position shifted w.r.t cps
+			errs[idx+1] = invokeServiceCmd(remotePeers[idx], cmd)
+		}(i)
+	}
+	wg.Wait()
+	errs[0] = invokeServiceCmd(cps[0], cmd)
+}
+
+// listPeerLocksInfo - fetch list of locks held on the given bucket,
+// matching prefix held longer than duration from all peer servers.
+func listPeerLocksInfo(peers adminPeers, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
+	// Used to aggregate volume lock information from all nodes.
+	allLocks := make([][]VolumeLockInfo, len(peers))
+	errs := make([]error, len(peers))
+	var wg sync.WaitGroup
+	localPeer := peers[0]
+	remotePeers := peers[1:]
+	for i, remotePeer := range remotePeers {
+		wg.Add(1)
+		go func(idx int, remotePeer adminPeer) {
+			defer wg.Done()
+			// `remotePeers` is right-shifted by one position relative to `peers`
+			allLocks[idx], errs[idx] = remotePeer.cmdRunner.ListLocks(bucket, prefix, duration)
+		}(i+1, remotePeer)
+	}
+	wg.Wait()
+	allLocks[0], errs[0] = localPeer.cmdRunner.ListLocks(bucket, prefix, duration)
+
+	// Summarizing errors received for ListLocks RPC across all
+	// nodes.  N B the possible unavailability of quorum in errors
+	// applies only to distributed setup.
+	errCount, err := reduceErrs(errs, []error{})
+	if err != nil {
+		if errCount >= (len(peers)/2 + 1) {
+			return nil, err
+		}
+		return nil, InsufficientReadQuorum{}
+	}
+
+	// Group lock information across nodes by (bucket, object)
+	// pair. For readability only.
+	paramLockMap := make(map[nsParam][]VolumeLockInfo)
+	for _, nodeLocks := range allLocks {
+		for _, lockInfo := range nodeLocks {
+			param := nsParam{
+				volume: lockInfo.Bucket,
+				path:   lockInfo.Object,
+			}
+			paramLockMap[param] = append(paramLockMap[param], lockInfo)
+		}
+	}
+	groupedLockInfos := []VolumeLockInfo{}
+	for _, volLocks := range paramLockMap {
+		groupedLockInfos = append(groupedLockInfos, volLocks...)
+	}
+	return groupedLockInfos, nil
+}
+
+// uptimeSlice - used to sort uptimes in chronological order.
+type uptimeSlice []struct {
+	err    error
+	uptime time.Duration
+}
+
+func (ts uptimeSlice) Len() int {
+	return len(ts)
+}
+
+func (ts uptimeSlice) Less(i, j int) bool {
+	return ts[i].uptime < ts[j].uptime
+}
+
+func (ts uptimeSlice) Swap(i, j int) {
+	ts[i], ts[j] = ts[j], ts[i]
+}
+
+// getPeerUptimes - returns the uptime since the last time read quorum
+// was established on success. Otherwise returns errXLReadQuorum.
+func getPeerUptimes(peers adminPeers) (time.Duration, error) {
+	// In a single node Erasure or FS backend setup the uptime of
+	// the setup is the uptime of the single minio server
+	// instance.
+	if !globalIsDistXL {
+		return UTCNow().Sub(globalBootTime), nil
+	}
+
+	uptimes := make(uptimeSlice, len(peers))
+
+	// Get up time of all servers.
+	wg := sync.WaitGroup{}
+	for i, peer := range peers {
+		wg.Add(1)
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+			serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
+			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
+		}(i, peer)
+	}
+	wg.Wait()
+
+	// Sort uptimes in chronological order.
+	sort.Sort(uptimes)
+
+	// Pick the readQuorum'th uptime in chronological order. i.e,
+	// the time at which read quorum was (re-)established.
+	readQuorum := len(uptimes) / 2
+	validCount := 0
+	latestUptime := time.Duration(0)
+	for _, uptime := range uptimes {
+		if uptime.err != nil {
+			logger.LogIf(context.Background(), uptime.err)
+			continue
+		}
+
+		validCount++
+		if validCount >= readQuorum {
+			latestUptime = uptime.uptime
+			break
+		}
+	}
+
+	// Less than readQuorum "Admin.Uptime" RPC call returned
+	// successfully, so read-quorum unavailable.
+	if validCount < readQuorum {
+		return time.Duration(0), InsufficientReadQuorum{}
+	}
+
+	return latestUptime, nil
+}
+
+// getPeerConfig - Fetches config.json from all nodes in the setup and
+// returns the one that occurs in a majority of them.
+func getPeerConfig(peers adminPeers) ([]byte, error) {
+	if !globalIsDistXL {
+		return peers[0].cmdRunner.GetConfig()
+	}
+
+	errs := make([]error, len(peers))
+	configs := make([][]byte, len(peers))
+
+	// Get config from all servers.
+	wg := sync.WaitGroup{}
+	for i, peer := range peers {
+		wg.Add(1)
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+			configs[idx], errs[idx] = peer.cmdRunner.GetConfig()
+		}(i, peer)
+	}
+	wg.Wait()
+
+	// Find the maximally occurring config among peers in a
+	// distributed setup.
+
+	serverConfigs := make([]serverConfig, len(peers))
+	for i, configBytes := range configs {
+		if errs[i] != nil {
+			continue
+		}
+
+		// Unmarshal the received config files.
+		err := json.Unmarshal(configBytes, &serverConfigs[i])
+		if err != nil {
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peers[i].addr)
+			ctx := logger.SetReqInfo(context.Background(), reqInfo)
+			logger.LogIf(ctx, err)
+			return nil, err
+		}
+	}
+
+	configJSON, err := getValidServerConfig(serverConfigs, errs)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return nil, err
+	}
+
+	// Return the config.json that was present quorum or more
+	// number of disks.
+	return json.Marshal(configJSON)
+}
+
+// getValidServerConfig - finds the server config that is present in
+// quorum or more number of servers.
+func getValidServerConfig(serverConfigs []serverConfig, errs []error) (scv serverConfig, e error) {
+	// majority-based quorum
+	quorum := len(serverConfigs)/2 + 1
+
+	// Count the number of disks a config.json was found in.
+	configCounter := make([]int, len(serverConfigs))
+
+	// We group equal serverConfigs by the lowest index of the
+	// same value;  e.g, let us take the following serverConfigs
+	// in a 4-node setup,
+	// serverConfigs == [c1, c2, c1, c1]
+	// configCounter == [3, 1, 0, 0]
+	// c1, c2 are the only distinct values that appear.  c1 is
+	// identified by 0, the lowest index it appears in and c2 is
+	// identified by 1. So, we need to find the number of times
+	// each of these distinct values occur.
+
+	// Invariants:
+
+	// 1. At the beginning of the i-th iteration, the number of
+	// unique configurations seen so far is equal to the number of
+	// non-zero counter values in config[:i].
+
+	// 2. At the beginning of the i-th iteration, the sum of
+	// elements of configCounter[:i] is equal to the number of
+	// non-error configurations seen so far.
+
+	// For each of the serverConfig ...
+	for i := range serverConfigs {
+		// Skip nodes where getConfig failed.
+		if errs[i] != nil {
+			continue
+		}
+		// Check if it is equal to any of the configurations
+		// seen so far. If j == i is reached then we have an
+		// unseen configuration.
+		for j := 0; j <= i; j++ {
+			if j < i && configCounter[j] == 0 {
+				// serverConfigs[j] is known to be
+				// equal to a value that was already
+				// seen. See example above for
+				// clarity.
+				continue
+			} else if j < i && serverConfigs[i].ConfigDiff(&serverConfigs[j]) == "" {
+				// serverConfigs[i] is equal to
+				// serverConfigs[j], update
+				// serverConfigs[j]'s counter since it
+				// is the lower index.
+				configCounter[j]++
+				break
+			} else if j == i {
+				// serverConfigs[i] is equal to no
+				// other value seen before. It is
+				// unique so far.
+				configCounter[i] = 1
+				break
+			} // else invariants specified above are violated.
+		}
+	}
+
+	// We find the maximally occurring server config and check if
+	// there is quorum.
+	var configJSON serverConfig
+	maxOccurrence := 0
+	for i, count := range configCounter {
+		if maxOccurrence < count {
+			maxOccurrence = count
+			configJSON = serverConfigs[i]
+		}
+	}
+
+	// If quorum nodes don't agree.
+	if maxOccurrence < quorum {
+		return scv, errXLWriteQuorum
+	}
+
+	return configJSON, nil
+}
+
+// Write config contents into a temporary file on all nodes.
+func writeTmpConfigPeers(peers adminPeers, tmpFileName string, configBytes []byte) []error {
+	// For a single-node minio server setup.
+	if !globalIsDistXL {
+		err := peers[0].cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
+		return []error{err}
+	}
+
+	errs := make([]error, len(peers))
+
+	// Write config into temporary file on all nodes.
+	wg := sync.WaitGroup{}
+	for i, peer := range peers {
+		wg.Add(1)
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+			errs[idx] = peer.cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
+		}(i, peer)
+	}
+	wg.Wait()
+
+	// Return bytes written and errors (if any) during writing
+	// temporary config file.
+	return errs
+}
+
+// Move config contents from the given temporary file onto config.json
+// on all nodes.
+func commitConfigPeers(peers adminPeers, tmpFileName string) []error {
+	// For a single-node minio server setup.
+	if !globalIsDistXL {
+		return []error{peers[0].cmdRunner.CommitConfig(tmpFileName)}
+	}
+
+	errs := make([]error, len(peers))
+
+	// Rename temporary config file into configDir/config.json on
+	// all nodes.
+	wg := sync.WaitGroup{}
+	for i, peer := range peers {
+		wg.Add(1)
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+			errs[idx] = peer.cmdRunner.CommitConfig(tmpFileName)
+		}(i, peer)
+	}
+	wg.Wait()
+
+	// Return errors (if any) received during rename.
+	return errs
+}

cmd/admin-rpc-client_test.go

@@ -1,260 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"encoding/json"
-	"reflect"
-	"testing"
-)
-
-var (
-	config1 = []byte(`{
-	"version": "13",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "us-east-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "debug"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	}
-}
-`)
-	// diff from config1 - amqp.Enable is True
-	config2 = []byte(`{
-	"version": "13",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "us-east-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "debug"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": true,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	}
-}
-`)
-)
-
-// TestGetValidServerConfig - test for getValidServerConfig.
-func TestGetValidServerConfig(t *testing.T) {
-	var c1, c2 serverConfig
-	err := json.Unmarshal(config1, &c1)
-	if err != nil {
-		t.Fatalf("json unmarshal of %s failed: %v", string(config1), err)
-	}
-
-	err = json.Unmarshal(config2, &c2)
-	if err != nil {
-		t.Fatalf("json unmarshal of %s failed: %v", string(config2), err)
-	}
-
-	// Valid config.
-	noErrs := []error{nil, nil, nil, nil}
-	serverConfigs := []serverConfig{c1, c2, c1, c1}
-	validConfig, err := getValidServerConfig(serverConfigs, noErrs)
-	if err != nil {
-		t.Errorf("Expected a valid config but received %v instead", err)
-	}
-
-	if !reflect.DeepEqual(validConfig, c1) {
-		t.Errorf("Expected valid config to be %v but received %v", config1, validConfig)
-	}
-
-	// Invalid config - no quorum.
-	serverConfigs = []serverConfig{c1, c2, c2, c1}
-	_, err = getValidServerConfig(serverConfigs, noErrs)
-	if err != errXLWriteQuorum {
-		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
-	}
-
-	// All errors
-	allErrs := []error{errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound}
-	serverConfigs = []serverConfig{{}, {}, {}, {}}
-	_, err = getValidServerConfig(serverConfigs, allErrs)
-	if err != errXLWriteQuorum {
-		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
-	}
-}

cmd/admin-rpc-server.go

@@ -18,211 +18,110 @@ package cmd
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
+	"path"
 	"time"
 
-	router "github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	xrpc "github.com/minio/minio/cmd/rpc"
 )
 
-const adminPath = "/admin"
+const adminServiceName = "Admin"
+const adminServiceSubPath = "/admin"
 
-// adminCmd - exports RPC methods for service status, stop and
-// restart commands.
-type adminCmd struct {
-	AuthRPCServer
+var adminServicePath = path.Join(minioReservedBucketPath, adminServiceSubPath)
+
+// adminRPCReceiver - Admin RPC receiver for admin RPC server.
+type adminRPCReceiver struct {
+	local *localAdminClient
 }
 
 // SignalServiceArgs - provides the signal argument to SignalService RPC
 type SignalServiceArgs struct {
-	AuthRPCArgs
+	AuthArgs
 	Sig serviceSignal
 }
 
+// SignalService - Send a restart or stop signal to the service
+func (receiver *adminRPCReceiver) SignalService(args *SignalServiceArgs, reply *VoidReply) error {
+	return receiver.local.SignalService(args.Sig)
+}
+
 // ListLocksQuery - wraps ListLocks API's query values to send over RPC.
 type ListLocksQuery struct {
-	AuthRPCArgs
+	AuthArgs
 	Bucket   string
 	Prefix   string
 	Duration time.Duration
 }
 
-// ListLocksReply - wraps ListLocks response over RPC.
-type ListLocksReply struct {
-	AuthRPCReply
-	VolLocks []VolumeLockInfo
+// ListLocks - lists locks held by requests handled by this server instance.
+func (receiver *adminRPCReceiver) ListLocks(args *ListLocksQuery, reply *[]VolumeLockInfo) (err error) {
+	*reply, err = receiver.local.ListLocks(args.Bucket, args.Prefix, args.Duration)
+	return err
 }
 
-// ServerInfoDataReply - wraps the server info response over RPC.
-type ServerInfoDataReply struct {
-	AuthRPCReply
-	ServerInfoData ServerInfoData
+// ServerInfo - returns the server info when object layer was initialized on this server.
+func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoData) (err error) {
+	*reply, err = receiver.local.ServerInfo()
+	return err
 }
 
-// ConfigReply - wraps the server config response over RPC.
-type ConfigReply struct {
-	AuthRPCReply
-	Config []byte // json-marshalled bytes of serverConfigV13
-}
-
-// SignalService - Send a restart or stop signal to the service
-func (s *adminCmd) SignalService(args *SignalServiceArgs, reply *AuthRPCReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	globalServiceSignalCh <- args.Sig
-	return nil
+// GetConfig - returns the config.json of this server.
+func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
+	*reply, err = receiver.local.GetConfig()
+	return err
 }
 
 // ReInitFormatArgs - provides dry-run information to re-initialize format.json
 type ReInitFormatArgs struct {
-	AuthRPCArgs
+	AuthArgs
 	DryRun bool
 }
 
 // ReInitFormat - re-init 'format.json'
-func (s *adminCmd) ReInitFormat(args *ReInitFormatArgs, reply *AuthRPCReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	_, err := objectAPI.HealFormat(context.Background(), args.DryRun)
-	return err
-}
-
-// ListLocks - lists locks held by requests handled by this server instance.
-func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error {
-	if err := query.IsAuthenticated(); err != nil {
-		return err
-	}
-	volLocks := listLocksInfo(query.Bucket, query.Prefix, query.Duration)
-	*reply = ListLocksReply{VolLocks: volLocks}
-	return nil
-}
-
-// ServerInfo - returns the server info when object layer was initialized on this server.
-func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	if globalBootTime.IsZero() {
-		return errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return errServerNotInitialized
-	}
-	storageInfo := objLayer.StorageInfo(context.Background())
-
-	reply.ServerInfoData = ServerInfoData{
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			Region:   globalServerConfig.GetRegion(),
-			SQSARN:   globalNotificationSys.GetARNList(),
-		},
-		StorageInfo: storageInfo,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-	}
-
-	return nil
-}
-
-// GetConfig - returns the config.json of this server.
-func (s *adminCmd) GetConfig(args *AuthRPCArgs, reply *ConfigReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	if globalServerConfig == nil {
-		return fmt.Errorf("config not present")
-	}
-
-	jsonBytes, err := json.Marshal(globalServerConfig)
-	if err != nil {
-		return err
-	}
-
-	reply.Config = jsonBytes
-	return nil
+func (receiver *adminRPCReceiver) ReInitFormat(args *ReInitFormatArgs, reply *VoidReply) error {
+	return receiver.local.ReInitFormat(args.DryRun)
 }
 
 // WriteConfigArgs - wraps the bytes to be written and temporary file name.
 type WriteConfigArgs struct {
-	AuthRPCArgs
+	AuthArgs
 	TmpFileName string
 	Buf         []byte
 }
 
-// WriteConfigReply - wraps the result of a writing config into a temporary file.
-// the remote node.
-type WriteConfigReply struct {
-	AuthRPCReply
-}
-
-func writeTmpConfigCommon(tmpFileName string, configBytes []byte) error {
-	tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName)
-	err := ioutil.WriteFile(tmpConfigFile, configBytes, 0666)
-	errorIf(err, fmt.Sprintf("Failed to write to temporary config file %s", tmpConfigFile))
-	return err
-}
-
 // WriteTmpConfig - writes the supplied config contents onto the
 // supplied temporary file.
-func (s *adminCmd) WriteTmpConfig(wArgs *WriteConfigArgs, wReply *WriteConfigReply) error {
-	if err := wArgs.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	return writeTmpConfigCommon(wArgs.TmpFileName, wArgs.Buf)
+func (receiver *adminRPCReceiver) WriteTmpConfig(args *WriteConfigArgs, reply *VoidReply) error {
+	return receiver.local.WriteTmpConfig(args.TmpFileName, args.Buf)
 }
 
 // CommitConfigArgs - wraps the config file name that needs to be
 // committed into config.json on this node.
 type CommitConfigArgs struct {
-	AuthRPCArgs
+	AuthArgs
 	FileName string
 }
 
-// CommitConfigReply - represents response to commit of config file on
-// this node.
-type CommitConfigReply struct {
-	AuthRPCReply
-}
-
 // CommitConfig - Renames the temporary file into config.json on this node.
-func (s *adminCmd) CommitConfig(cArgs *CommitConfigArgs, cReply *CommitConfigReply) error {
-	configFile := getConfigFile()
-	tmpConfigFile := filepath.Join(getConfigDir(), cArgs.FileName)
-
-	err := os.Rename(tmpConfigFile, configFile)
-	errorIf(err, fmt.Sprintf("Failed to rename %s to %s", tmpConfigFile, configFile))
-	return err
+func (receiver *adminRPCReceiver) CommitConfig(args *CommitConfigArgs, reply *VoidReply) error {
+	return receiver.local.CommitConfig(args.FileName)
 }
 
-// registerAdminRPCRouter - registers RPC methods for service status,
-// stop and restart commands.
-func registerAdminRPCRouter(mux *router.Router) error {
-	adminRPCHandler := &adminCmd{}
-	adminRPCServer := newRPCServer()
-	err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
-	if err != nil {
-		return errors.Trace(err)
+// NewAdminRPCServer - returns new admin RPC server.
+func NewAdminRPCServer() (*xrpc.Server, error) {
+	rpcServer := xrpc.NewServer()
+	if err := rpcServer.RegisterName(adminServiceName, &adminRPCReceiver{&localAdminClient{}}); err != nil {
+		return nil, err
 	}
-	adminRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
-	adminRouter.Path(adminPath).Handler(adminRPCServer)
-	return nil
+	return rpcServer, nil
+}
+
+// registerAdminRPCRouter - creates and registers Admin RPC server and its router.
+func registerAdminRPCRouter(router *mux.Router) {
+	rpcServer, err := NewAdminRPCServer()
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
+	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
+	subrouter.Path(adminServiceSubPath).Handler(rpcServer)
 }

cmd/admin-rpc-server_test.go

@@ -1,262 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"encoding/json"
-	"os"
-	"testing"
-)
-
-func testAdminCmd(cmd cmdType, t *testing.T) {
-	// reset globals. this is to make sure that the tests are not
-	// affected by modified globals.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatalf("Failed to create test config - %v", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	adminServer := adminCmd{}
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	err = adminServer.Login(&args, &LoginRPCReply{})
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	go func() {
-		// A test signal receiver
-		<-globalServiceSignalCh
-	}()
-
-	sa := SignalServiceArgs{
-		AuthRPCArgs: AuthRPCArgs{AuthToken: token, Version: globalRPCAPIVersion},
-		Sig:         cmd.toServiceSignal(),
-	}
-
-	genReply := AuthRPCReply{}
-	switch cmd {
-	case restartCmd, stopCmd:
-		if err = adminServer.SignalService(&sa, &genReply); err != nil {
-			t.Errorf("restartCmd/stopCmd: Expected: <nil>, got: %v",
-				err)
-		}
-	default:
-		err = adminServer.SignalService(&sa, &genReply)
-		if err != nil && err.Error() != errUnsupportedSignal.Error() {
-			t.Errorf("invalidSignal %s: unexpected error got: %v",
-				cmd, err)
-		}
-	}
-}
-
-// TestAdminRestart - test for Admin.Restart RPC service.
-func TestAdminRestart(t *testing.T) {
-	testAdminCmd(restartCmd, t)
-}
-
-// TestAdminStop - test for Admin.Stop RPC service.
-func TestAdminStop(t *testing.T) {
-	testAdminCmd(stopCmd, t)
-}
-
-// TestAdminStatus - test for Admin.Status RPC service (error case)
-func TestAdminStatus(t *testing.T) {
-	testAdminCmd(statusCmd, t)
-}
-
-// TestReInitFormat - test for Admin.ReInitFormat RPC service.
-func TestReInitFormat(t *testing.T) {
-	// Reset global variables to start afresh.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig("us-east-1")
-	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	// Initializing objectLayer for HealFormatHandler.
-	_, xlDirs, xlErr := initTestXLObjLayer()
-	if xlErr != nil {
-		t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
-	}
-	defer removeRoots(xlDirs)
-
-	// Set globalEndpoints for a single node XL setup.
-	globalEndpoints = mustGetNewEndpointList(xlDirs...)
-
-	// Setup admin rpc server for an XL backend.
-	globalIsXL = true
-	adminServer := adminCmd{}
-
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	err = adminServer.Login(&args, &LoginRPCReply{})
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	authArgs := AuthRPCArgs{
-		AuthToken: token,
-		Version:   globalRPCAPIVersion,
-	}
-	authReply := AuthRPCReply{}
-
-	err = adminServer.ReInitFormat(&ReInitFormatArgs{
-		AuthRPCArgs: authArgs,
-		DryRun:      false,
-	}, &authReply)
-	if err != nil {
-		t.Errorf("Expected to pass, but failed with %v", err)
-	}
-}
-
-// TestGetConfig - Test for GetConfig admin RPC.
-func TestGetConfig(t *testing.T) {
-	// Reset global variables to start afresh.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig("us-east-1")
-	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	adminServer := adminCmd{}
-	creds := globalServerConfig.GetCredential()
-
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	reply := LoginRPCReply{}
-	err = adminServer.Login(&args, &reply)
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	authArgs := AuthRPCArgs{
-		AuthToken: token,
-		Version:   globalRPCAPIVersion,
-	}
-
-	configReply := ConfigReply{}
-
-	err = adminServer.GetConfig(&authArgs, &configReply)
-	if err != nil {
-		t.Errorf("Expected GetConfig to pass but failed with %v", err)
-	}
-
-	var config serverConfigV13
-	err = json.Unmarshal(configReply.Config, &config)
-	if err != nil {
-		t.Errorf("Expected json unmarshal to pass but failed with %v", err)
-	}
-}
-
-// TestWriteAndCommitConfig - test for WriteTmpConfig and CommitConfig
-// RPC handler.
-func TestWriteAndCommitConfig(t *testing.T) {
-	// Reset global variables to start afresh.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig("us-east-1")
-	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	adminServer := adminCmd{}
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	reply := LoginRPCReply{}
-	err = adminServer.Login(&args, &reply)
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	// Write temporary config.
-	buf := []byte("hello")
-	tmpFileName := mustGetUUID()
-	wArgs := WriteConfigArgs{
-		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: token,
-			Version:   globalRPCAPIVersion,
-		},
-		TmpFileName: tmpFileName,
-		Buf:         buf,
-	}
-
-	err = adminServer.WriteTmpConfig(&wArgs, &WriteConfigReply{})
-	if err != nil {
-		t.Fatalf("Failed to write temporary config %v", err)
-	}
-
-	if err != nil {
-		t.Errorf("Expected to succeed but failed %v", err)
-	}
-
-	cArgs := CommitConfigArgs{
-		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: token,
-			Version:   globalRPCAPIVersion,
-		},
-		FileName: tmpFileName,
-	}
-	cReply := CommitConfigReply{}
-
-	err = adminServer.CommitConfig(&cArgs, &cReply)
-	if err != nil {
-		t.Fatalf("Failed to commit config file %v", err)
-	}
-}

cmd/admin-rpc_test.go

@@ -0,0 +1,613 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// localAdminClient and AdminRPCClient are adminCmdRunner interface compatible,
+// hence below test functions are available for both clients.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// Admin RPC server, adminRPCReceiver and AdminRPCClient are
+// inter-dependent, below test functions are sufficient to test all of them.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
+	tmpGlobalServiceSignalCh := globalServiceSignalCh
+	globalServiceSignalCh = make(chan serviceSignal, 10)
+	defer func() {
+		globalServiceSignalCh = tmpGlobalServiceSignalCh
+	}()
+
+	testCases := []struct {
+		signal    serviceSignal
+		expectErr bool
+	}{
+		{serviceRestart, false},
+		{serviceStop, false},
+		{serviceStatus, true},
+		{serviceSignal(100), true},
+	}
+
+	for i, testCase := range testCases {
+		err := client.SignalService(testCase.signal)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerReInitFormat(t *testing.T, client adminCmdRunner) {
+	tmpGlobalObjectAPI := globalObjectAPI
+	defer func() {
+		globalObjectAPI = tmpGlobalObjectAPI
+	}()
+
+	testCases := []struct {
+		objectAPI ObjectLayer
+		dryRun    bool
+		expectErr bool
+	}{
+		{&DummyObjectLayer{}, true, false},
+		{&DummyObjectLayer{}, false, false},
+		{nil, true, true},
+		{nil, false, true},
+	}
+
+	for i, testCase := range testCases {
+		globalObjectAPI = testCase.objectAPI
+		err := client.ReInitFormat(testCase.dryRun)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerListLocks(t *testing.T, client adminCmdRunner) {
+	tmpGlobalObjectAPI := globalObjectAPI
+	defer func() {
+		globalObjectAPI = tmpGlobalObjectAPI
+	}()
+
+	testCases := []struct {
+		objectAPI ObjectLayer
+		expectErr bool
+	}{
+		{&DummyObjectLayer{}, false},
+		{nil, true},
+	}
+
+	for i, testCase := range testCases {
+		globalObjectAPI = testCase.objectAPI
+		_, err := client.ListLocks("", "", time.Duration(0))
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
+	tmpGlobalBootTime := globalBootTime
+	tmpGlobalObjectAPI := globalObjectAPI
+	tmpGlobalConnStats := globalConnStats
+	tmpGlobalHTTPStats := globalHTTPStats
+	tmpGlobalNotificationSys := globalNotificationSys
+	defer func() {
+		globalBootTime = tmpGlobalBootTime
+		globalObjectAPI = tmpGlobalObjectAPI
+		globalConnStats = tmpGlobalConnStats
+		globalHTTPStats = tmpGlobalHTTPStats
+		globalNotificationSys = tmpGlobalNotificationSys
+	}()
+
+	endpoints := new(EndpointList)
+	notificationSys, err := NewNotificationSys(globalServerConfig, *endpoints)
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	testCases := []struct {
+		bootTime        time.Time
+		objectAPI       ObjectLayer
+		connStats       *ConnStats
+		httpStats       *HTTPStats
+		notificationSys *NotificationSys
+		expectErr       bool
+	}{
+		{UTCNow(), &DummyObjectLayer{}, newConnStats(), newHTTPStats(), notificationSys, false},
+		{time.Time{}, nil, nil, nil, nil, true},
+		{UTCNow(), nil, nil, nil, nil, true},
+	}
+
+	for i, testCase := range testCases {
+		globalBootTime = testCase.bootTime
+		globalObjectAPI = testCase.objectAPI
+		globalConnStats = testCase.connStats
+		globalHTTPStats = testCase.httpStats
+		globalNotificationSys = testCase.notificationSys
+		_, err := client.ServerInfo()
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerGetConfig(t *testing.T, client adminCmdRunner) {
+	tmpGlobalServerConfig := globalServerConfig
+	defer func() {
+		globalServerConfig = tmpGlobalServerConfig
+	}()
+
+	config := newServerConfig()
+
+	testCases := []struct {
+		config    *serverConfig
+		expectErr bool
+	}{
+		{globalServerConfig, false},
+		{config, false},
+	}
+
+	for i, testCase := range testCases {
+		globalServerConfig = testCase.config
+		_, err := client.GetConfig()
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerWriteTmpConfig(t *testing.T, client adminCmdRunner) {
+	tmpConfigDir := configDir
+	defer func() {
+		configDir = tmpConfigDir
+	}()
+
+	tempDir, err := ioutil.TempDir("", ".AdminCmdRunnerWriteTmpConfig.")
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+	configDir = &ConfigDir{dir: tempDir}
+
+	testCases := []struct {
+		tmpFilename string
+		configBytes []byte
+		expectErr   bool
+	}{
+		{"config1.json", []byte(`{"version":"23","region":"us-west-1a"}`), false},
+		// Overwrite test.
+		{"config1.json", []byte(`{"version":"23","region":"us-west-1a","browser":"on"}`), false},
+		{"config2.json", []byte{}, false},
+		{"config3.json", nil, false},
+	}
+
+	for i, testCase := range testCases {
+		err := client.WriteTmpConfig(testCase.tmpFilename, testCase.configBytes)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerCommitConfig(t *testing.T, client adminCmdRunner) {
+	tmpConfigDir := configDir
+	defer func() {
+		configDir = tmpConfigDir
+	}()
+
+	tempDir, err := ioutil.TempDir("", ".AdminCmdRunnerCommitConfig.")
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+	configDir = &ConfigDir{dir: tempDir}
+	err = ioutil.WriteFile(filepath.Join(tempDir, "config.json"), []byte{}, os.ModePerm)
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	err = client.WriteTmpConfig("config1.json", []byte(`{"version":"23","region":"us-west-1a"}`))
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	testCases := []struct {
+		tmpFilename string
+		expectErr   bool
+	}{
+		{"config1.json", false},
+		{"config2.json", true},
+	}
+
+	for i, testCase := range testCases {
+		err := client.CommitConfig(testCase.tmpFilename)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func newAdminRPCHTTPServerClient(t *testing.T) (*httptest.Server, *AdminRPCClient, *serverConfig) {
+	rpcServer, err := NewAdminRPCServer()
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		rpcServer.ServeHTTP(w, r)
+	}))
+
+	url, err := xnet.ParseURL(httpServer.URL)
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	host, err := xnet.ParseHost(url.Host)
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	prevGlobalServerConfig := globalServerConfig
+	globalServerConfig = newServerConfig()
+
+	rpcClient, err := NewAdminRPCClient(host)
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	return httpServer, rpcClient, prevGlobalServerConfig
+}
+
+func TestAdminRPCClientSignalService(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerSignalService(t, rpcClient)
+}
+
+func TestAdminRPCClientReInitFormat(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerReInitFormat(t, rpcClient)
+}
+
+func TestAdminRPCClientListLocks(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerListLocks(t, rpcClient)
+}
+
+func TestAdminRPCClientServerInfo(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerServerInfo(t, rpcClient)
+}
+
+func TestAdminRPCClientGetConfig(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerGetConfig(t, rpcClient)
+}
+
+func TestAdminRPCClientWriteTmpConfig(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerWriteTmpConfig(t, rpcClient)
+}
+
+func TestAdminRPCClientCommitConfig(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerCommitConfig(t, rpcClient)
+}
+
+var (
+	config1 = []byte(`{
+	"version": "13",
+	"credential": {
+		"accessKey": "minio",
+		"secretKey": "minio123"
+	},
+	"region": "us-east-1",
+	"logger": {
+		"console": {
+			"enable": true,
+			"level": "debug"
+		},
+		"file": {
+			"enable": false,
+			"fileName": "",
+			"level": ""
+		}
+	},
+	"notify": {
+		"amqp": {
+			"1": {
+				"enable": false,
+				"url": "",
+				"exchange": "",
+				"routingKey": "",
+				"exchangeType": "",
+				"mandatory": false,
+				"immediate": false,
+				"durable": false,
+				"internal": false,
+				"noWait": false,
+				"autoDeleted": false
+			}
+		},
+		"nats": {
+			"1": {
+				"enable": false,
+				"address": "",
+				"subject": "",
+				"username": "",
+				"password": "",
+				"token": "",
+				"secure": false,
+				"pingInterval": 0,
+				"streaming": {
+					"enable": false,
+					"clusterID": "",
+					"clientID": "",
+					"async": false,
+					"maxPubAcksInflight": 0
+				}
+			}
+		},
+		"elasticsearch": {
+			"1": {
+				"enable": false,
+				"url": "",
+				"index": ""
+			}
+		},
+		"redis": {
+			"1": {
+				"enable": false,
+				"address": "",
+				"password": "",
+				"key": ""
+			}
+		},
+		"postgresql": {
+			"1": {
+				"enable": false,
+				"connectionString": "",
+				"table": "",
+				"host": "",
+				"port": "",
+				"user": "",
+				"password": "",
+				"database": ""
+			}
+		},
+		"kafka": {
+			"1": {
+				"enable": false,
+				"brokers": null,
+				"topic": ""
+			}
+		},
+		"webhook": {
+			"1": {
+				"enable": false,
+				"endpoint": ""
+			}
+		}
+	}
+}
+`)
+	// diff from config1 - amqp.Enable is True
+	config2 = []byte(`{
+	"version": "13",
+	"credential": {
+		"accessKey": "minio",
+		"secretKey": "minio123"
+	},
+	"region": "us-east-1",
+	"logger": {
+		"console": {
+			"enable": true,
+			"level": "debug"
+		},
+		"file": {
+			"enable": false,
+			"fileName": "",
+			"level": ""
+		}
+	},
+	"notify": {
+		"amqp": {
+			"1": {
+				"enable": true,
+				"url": "",
+				"exchange": "",
+				"routingKey": "",
+				"exchangeType": "",
+				"mandatory": false,
+				"immediate": false,
+				"durable": false,
+				"internal": false,
+				"noWait": false,
+				"autoDeleted": false
+			}
+		},
+		"nats": {
+			"1": {
+				"enable": false,
+				"address": "",
+				"subject": "",
+				"username": "",
+				"password": "",
+				"token": "",
+				"secure": false,
+				"pingInterval": 0,
+				"streaming": {
+					"enable": false,
+					"clusterID": "",
+					"clientID": "",
+					"async": false,
+					"maxPubAcksInflight": 0
+				}
+			}
+		},
+		"elasticsearch": {
+			"1": {
+				"enable": false,
+				"url": "",
+				"index": ""
+			}
+		},
+		"redis": {
+			"1": {
+				"enable": false,
+				"address": "",
+				"password": "",
+				"key": ""
+			}
+		},
+		"postgresql": {
+			"1": {
+				"enable": false,
+				"connectionString": "",
+				"table": "",
+				"host": "",
+				"port": "",
+				"user": "",
+				"password": "",
+				"database": ""
+			}
+		},
+		"kafka": {
+			"1": {
+				"enable": false,
+				"brokers": null,
+				"topic": ""
+			}
+		},
+		"webhook": {
+			"1": {
+				"enable": false,
+				"endpoint": ""
+			}
+		}
+	}
+}
+`)
+)
+
+// TestGetValidServerConfig - test for getValidServerConfig.
+func TestGetValidServerConfig(t *testing.T) {
+	var c1, c2 serverConfig
+	err := json.Unmarshal(config1, &c1)
+	if err != nil {
+		t.Fatalf("json unmarshal of %s failed: %v", string(config1), err)
+	}
+
+	err = json.Unmarshal(config2, &c2)
+	if err != nil {
+		t.Fatalf("json unmarshal of %s failed: %v", string(config2), err)
+	}
+
+	// Valid config.
+	noErrs := []error{nil, nil, nil, nil}
+	serverConfigs := []serverConfig{c1, c2, c1, c1}
+	validConfig, err := getValidServerConfig(serverConfigs, noErrs)
+	if err != nil {
+		t.Errorf("Expected a valid config but received %v instead", err)
+	}
+
+	if !reflect.DeepEqual(validConfig, c1) {
+		t.Errorf("Expected valid config to be %v but received %v", config1, validConfig)
+	}
+
+	// Invalid config - no quorum.
+	serverConfigs = []serverConfig{c1, c2, c2, c1}
+	_, err = getValidServerConfig(serverConfigs, noErrs)
+	if err != errXLWriteQuorum {
+		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
+	}
+
+	// All errors
+	allErrs := []error{errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound}
+	serverConfigs = []serverConfig{{}, {}, {}, {}}
+	_, err = getValidServerConfig(serverConfigs, allErrs)
+	if err != errXLWriteQuorum {
+		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
+	}
+}

cmd/api-errors.go

@@ -17,12 +17,13 @@
 package cmd
 
 import (
+	"context"
 	"encoding/xml"
 	"fmt"
 	"net/http"
 
+	"github.com/coreos/etcd/client"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 )
@@ -138,6 +139,7 @@ const (
 	ErrMissingSSECustomerKey
 	ErrMissingSSECustomerKeyMD5
 	ErrSSECustomerKeyMD5Mismatch
+	ErrInvalidSSECustomerParameters
 
 	// Bucket notification related errors.
 	ErrEventNotification
@@ -191,6 +193,7 @@ const (
 	ErrHealMissingBucket
 	ErrHealAlreadyRunning
 	ErrHealOverlappingPaths
+	ErrIncorrectContinuationToken
 )
 
 // error code to APIError structure, these fields carry respective
@@ -629,7 +632,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInsecureSSECustomerRequest: {
 		Code:           "InvalidRequest",
-		Description:    errInsecureSSERequest.Error(),
+		Description:    "Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrSSEMultipartEncrypted: {
@@ -639,7 +642,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrSSEEncryptedObject: {
 		Code:           "InvalidRequest",
-		Description:    errEncryptedObject.Error(),
+		Description:    "The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidEncryptionParameters: {
@@ -649,27 +652,32 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInvalidSSECustomerAlgorithm: {
 		Code:           "InvalidArgument",
-		Description:    errInvalidSSEAlgorithm.Error(),
+		Description:    "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidSSECustomerKey: {
 		Code:           "InvalidArgument",
-		Description:    errInvalidSSEKey.Error(),
+		Description:    "The secret key was invalid for the specified algorithm.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMissingSSECustomerKey: {
 		Code:           "InvalidArgument",
-		Description:    errMissingSSEKey.Error(),
+		Description:    "Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMissingSSECustomerKeyMD5: {
 		Code:           "InvalidArgument",
-		Description:    errMissingSSEKeyMD5.Error(),
+		Description:    "Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrSSECustomerKeyMD5Mismatch: {
 		Code:           "InvalidArgument",
-		Description:    errSSEKeyMD5Mismatch.Error(),
+		Description:    "The calculated MD5 hash of the key did not match the hash that was provided.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidSSECustomerParameters: {
+		Code:           "InvalidArgument",
+		Description:    "The provided encryption parameters did not match the ones used originally.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 
@@ -793,7 +801,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	// Generic Invalid-Request error. Should be used for response errors only for unlikely
-	// corner case errors for which introducing new APIErrorCode is not worth it. errorIf()
+	// corner case errors for which introducing new APIErrorCode is not worth it. LogIf()
 	// should be used to log the error at the source of the error for debugging purposes.
 	ErrInvalidRequest: {
 		Code:           "InvalidRequest",
@@ -835,7 +843,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Object storage backend is unreachable",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
-
+	ErrIncorrectContinuationToken: {
+		Code:           "InvalidArgument",
+		Description:    "The continuation token provided is incorrect",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 
@@ -847,7 +859,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
-	err = errors.Cause(err)
 	// Verify if the underlying error is signature mismatch.
 	switch err {
 	case errSignatureMismatch:
@@ -860,6 +871,29 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
+	// SSE errors
+	case errInsecureSSERequest:
+		apiErr = ErrInsecureSSECustomerRequest
+	case errInvalidSSEAlgorithm:
+		apiErr = ErrInvalidSSECustomerAlgorithm
+	case errInvalidSSEKey:
+		apiErr = ErrInvalidSSECustomerKey
+	case errMissingSSEKey:
+		apiErr = ErrMissingSSECustomerKey
+	case errMissingSSEKeyMD5:
+		apiErr = ErrMissingSSECustomerKeyMD5
+	case errSSEKeyMD5Mismatch:
+		apiErr = ErrSSECustomerKeyMD5Mismatch
+	case errObjectTampered:
+		apiErr = ErrObjectTampered
+	case errEncryptedObject:
+		apiErr = ErrSSEEncryptedObject
+	case errInvalidSSEParameters:
+		apiErr = ErrInvalidSSECustomerParameters
+	case errSSEKeyMismatch:
+		apiErr = ErrAccessDenied // no access without correct key
+	case context.Canceled, context.DeadlineExceeded:
+		apiErr = ErrOperationTimedOut
 	}
 
 	if apiErr != ErrNone {
@@ -867,25 +901,12 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return apiErr
 	}
 
-	switch err { // SSE errors
-	case errInsecureSSERequest:
-		return ErrInsecureSSECustomerRequest
-	case errInvalidSSEAlgorithm:
-		return ErrInvalidSSECustomerAlgorithm
-	case errInvalidSSEKey:
-		return ErrInvalidSSECustomerKey
-	case errMissingSSEKey:
-		return ErrMissingSSECustomerKey
-	case errMissingSSEKeyMD5:
-		return ErrMissingSSECustomerKeyMD5
-	case errSSEKeyMD5Mismatch:
-		return ErrSSECustomerKeyMD5Mismatch
-	case errObjectTampered:
-		return ErrObjectTampered
-	case errEncryptedObject:
-		return ErrSSEEncryptedObject
-	case errSSEKeyMismatch:
-		return ErrAccessDenied // no access without correct key
+	// etcd specific errors, a key is always a bucket for us return
+	// ErrNoSuchBucket in such a case.
+	if e, ok := err.(*client.Error); ok {
+		if e.Code == client.ErrorCodeKeyNotFound {
+			return ErrNoSuchBucket
+		}
 	}
 
 	switch err.(type) {
@@ -947,8 +968,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooSmall
 	case NotImplemented:
 		apiErr = ErrNotImplemented
-	case PolicyNotFound:
-		apiErr = ErrNoSuchBucketPolicy
 	case PartTooBig:
 		apiErr = ErrEntityTooLarge
 	case UnsupportedMetadata:

cmd/api-resources.go

@@ -36,7 +36,17 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
 }
 
 // Parse bucket url queries for ListObjects V2.
-func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string) {
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	// The continuation-token cannot be empty.
+	if val, ok := values["continuation-token"]; ok {
+		if len(val[0]) == 0 {
+			errCode = ErrIncorrectContinuationToken
+			return
+		}
+	}
+
 	prefix = values.Get("prefix")
 	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")
@@ -78,13 +88,3 @@ func getObjectResources(values url.Values) (uploadID string, partNumberMarker, m
 	encodingType = values.Get("encoding-type")
 	return
 }
-
-// Validates filter values
-func validateFilterValues(values []string) (err APIErrorCode) {
-	for _, value := range values {
-		if !IsValidObjectPrefix(value) {
-			return ErrFilterValueInvalid
-		}
-	}
-	return ErrNone
-}

cmd/api-resources_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"net/url"
-	"strings"
 	"testing"
 )
 
@@ -30,6 +29,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		fetchOwner                           bool
 		maxKeys                              int
 		encodingType                         string
+		errCode                              APIErrorCode
 	}{
 		{
 			values: url.Values{
@@ -48,6 +48,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      100,
 			encodingType: "gzip",
+			errCode:      ErrNone,
 		},
 		{
 			values: url.Values{
@@ -65,11 +66,34 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      1000,
 			encodingType: "gzip",
+			errCode:      ErrNone,
+		},
+		{
+			values: url.Values{
+				"prefix":             []string{"photos/"},
+				"continuation-token": []string{""},
+				"start-after":        []string{"start-after"},
+				"delimiter":          []string{"/"},
+				"fetch-owner":        []string{"true"},
+				"encoding-type":      []string{"gzip"},
+			},
+			prefix:       "",
+			token:        "",
+			startAfter:   "",
+			delimiter:    "",
+			fetchOwner:   false,
+			maxKeys:      0,
+			encodingType: "",
+			errCode:      ErrIncorrectContinuationToken,
 		},
 	}
 
 	for i, testCase := range testCases {
-		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType := getListObjectsV2Args(testCase.values)
+		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(testCase.values)
+
+		if errCode != testCase.errCode {
+			t.Errorf("Test %d: Expected error code:%d, got %d", i+1, testCase.errCode, errCode)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}
@@ -189,38 +213,3 @@ func TestGetObjectsResources(t *testing.T) {
 		}
 	}
 }
-
-// Validates if filter values are correct
-func TestValidateFilterValues(t *testing.T) {
-	testCases := []struct {
-		values        []string
-		expectedError APIErrorCode
-	}{
-		{
-			values:        []string{""},
-			expectedError: ErrNone,
-		},
-		{
-			values:        []string{"", "prefix"},
-			expectedError: ErrNone,
-		},
-		{
-			values:        []string{strings.Repeat("a", 1025)},
-			expectedError: ErrFilterValueInvalid,
-		},
-		{
-			values:        []string{"a\\b"},
-			expectedError: ErrFilterValueInvalid,
-		},
-		{
-			values:        []string{string([]byte{0xff, 0xfe, 0xfd})},
-			expectedError: ErrFilterValueInvalid,
-		},
-	}
-
-	for i, testCase := range testCases {
-		if actualError := validateFilterValues(testCase.values); actualError != testCase.expectedError {
-			t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.expectedError, actualError)
-		}
-	}
-}

cmd/api-response.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/pkg/handlers"
@@ -293,8 +294,10 @@ func getObjectLocation(r *http.Request, domain, bucket, object string) string {
 	}
 	// If domain is set then we need to use bucket DNS style.
 	if domain != "" {
-		u.Host = bucket + "." + domain
-		u.Path = path.Join(slashSeparator, object)
+		if strings.Contains(r.Host, domain) {
+			u.Host = bucket + "." + r.Host
+			u.Path = path.Join(slashSeparator, object)
+		}
 	}
 	return u.String()
 }
@@ -339,7 +342,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
-		content.StorageClass = globalMinioDefaultStorageClass
+		content.StorageClass = object.StorageClass
 		content.Owner = owner
 		contents = append(contents, content)
 	}
@@ -385,7 +388,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
-		content.StorageClass = globalMinioDefaultStorageClass
+		content.StorageClass = object.StorageClass
 		content.Owner = owner
 		contents = append(contents, content)
 	}

cmd/api-router.go

@@ -19,7 +19,8 @@ package cmd
 import (
 	"net/http"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // objectAPIHandler implements and provides http handlers for S3 API.
@@ -29,13 +30,13 @@ type objectAPIHandlers struct {
 }
 
 // registerAPIRouter - registers S3 compatible APIs.
-func registerAPIRouter(mux *router.Router) {
+func registerAPIRouter(router *mux.Router) {
 	var err error
 	var cacheConfig = globalServerConfig.GetCacheConfig()
 	if len(cacheConfig.Drives) > 0 {
 		// initialize the new disk cache objects.
 		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
-		fatalIf(err, "Unable to initialize disk caching")
+		logger.FatalIf(err, "Unable to initialize disk caching")
 	}
 
 	// Initialize API.
@@ -45,8 +46,8 @@ func registerAPIRouter(mux *router.Router) {
 	}
 
 	// API Router
-	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
-	var routers []*router.Router
+	apiRouter := router.PathPrefix("/").Subrouter()
+	var routers []*mux.Router
 	if globalDomainName != "" {
 		routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
 	}
@@ -68,6 +69,8 @@ func registerAPIRouter(mux *router.Router) {
 		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
 		// AbortMultipartUpload
 		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// GetObjectACL - this is a dummy call.
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
 		// GetObject
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
 		// CopyObject
@@ -82,6 +85,10 @@ func registerAPIRouter(mux *router.Router) {
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
 		// GetBucketPolicy
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+
+		// GetBucketACL -- this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
+
 		// GetBucketNotification
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
 		// ListenBucketNotification

cmd/auth-handler.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015-2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,14 +18,18 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"strings"
 
-	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // Verify if request has JWT.
@@ -110,37 +114,77 @@ func getRequestAuthType(r *http.Request) authType {
 // It does not accept presigned or JWT or anonymous requests.
 func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
 	s3Err := ErrAccessDenied
-	if getRequestAuthType(r) == authTypeSigned { // we only support V4 (no presign)
+	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok && getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) { // we only support V4 (no presign) with auth. body
 		s3Err = isReqAuthenticated(r, region)
 	}
 	if s3Err != ErrNone {
-		errorIf(errors.New(getAPIError(s3Err).Description), "%s", dumpRequest(r))
+		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description))
 	}
 	return s3Err
 }
 
-func checkRequestAuthType(r *http.Request, bucket, policyAction, region string) APIErrorCode {
-	reqAuthType := getRequestAuthType(r)
+func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) APIErrorCode {
+	isOwner := true
+	accountName := globalServerConfig.GetCredential().AccessKey
 
-	switch reqAuthType {
+	switch getRequestAuthType(r) {
+	case authTypeUnknown:
+		return ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
-		// Signature V2 validation.
-		return isReqAuthenticatedV2(r)
-	case authTypeSigned, authTypePresigned:
-		return isReqAuthenticated(r, region)
-	}
-
-	if reqAuthType == authTypeAnonymous && policyAction != "" {
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
-		if err != nil {
-			return ErrInternalError
+		if errorCode := isReqAuthenticatedV2(r); errorCode != ErrNone {
+			return errorCode
 		}
-		return enforceBucketPolicy(bucket, policyAction, resource,
-			r.Referer(), handlers.GetSourceIP(r), r.URL.Query())
+	case authTypeSigned, authTypePresigned:
+		region := globalServerConfig.GetRegion()
+		switch action {
+		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
+			region = ""
+		}
+
+		if errorCode := isReqAuthenticated(r, region); errorCode != ErrNone {
+			return errorCode
+		}
+	default:
+		isOwner = false
+		accountName = ""
+	}
+
+	// LocationConstraint is valid only for CreateBucketAction.
+	var locationConstraint string
+	if action == policy.CreateBucketAction {
+		// To extract region from XML in request body, get copy of request body.
+		payload, err := ioutil.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return ErrMalformedXML
+		}
+
+		// Populate payload to extract location constraint.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+
+		var s3Error APIErrorCode
+		locationConstraint, s3Error = parseLocationConstraint(r)
+		if s3Error != ErrNone {
+			return s3Error
+		}
+
+		// Populate payload again to handle it in HTTP handler.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+	}
+
+	if globalPolicySys.IsAllowed(policy.Args{
+		AccountName:     accountName,
+		Action:          action,
+		BucketName:      bucketName,
+		ConditionValues: getConditionValues(r, locationConstraint),
+		IsOwner:         isOwner,
+		ObjectName:      objectName,
+	}) {
+		return ErrNone
 	}
 
-	// By default return ErrAccessDenied
 	return ErrAccessDenied
 }
 
@@ -166,59 +210,45 @@ func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode)
 
 // Verify if request has valid AWS Signature Version '4'.
 func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
-	if r == nil {
-		return ErrInternalError
-	}
-
 	if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
 		return errCode
 	}
 
-	payload, err := ioutil.ReadAll(r.Body)
+	var (
+		err                       error
+		contentMD5, contentSHA256 []byte
+	)
+	// Extract 'Content-Md5' if present.
+	if _, ok := r.Header["Content-Md5"]; ok {
+		contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get("Content-Md5"))
+		if err != nil || len(contentMD5) == 0 {
+			return ErrInvalidDigest
+		}
+	}
+
+	// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
+	// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
+	if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
+		if sha256Sum, ok := r.URL.Query()["X-Amz-Content-Sha256"]; ok && len(sha256Sum) > 0 {
+			contentSHA256, err = hex.DecodeString(sha256Sum[0])
+			if err != nil {
+				return ErrContentSHA256Mismatch
+			}
+		}
+	} else if _, ok := r.Header["X-Amz-Content-Sha256"]; !skipSHA256 && ok {
+		contentSHA256, err = hex.DecodeString(r.Header.Get("X-Amz-Content-Sha256"))
+		if err != nil || len(contentSHA256) == 0 {
+			return ErrContentSHA256Mismatch
+		}
+	}
+
+	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
+	// The verification happens implicit during reading.
+	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256))
 	if err != nil {
-		errorIf(err, "Unable to read request body for signature verification")
-		return ErrInternalError
-	}
-
-	// Populate back the payload.
-	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
-
-	// Verify Content-Md5, if payload is set.
-	if clntMD5B64, ok := r.Header["Content-Md5"]; ok {
-		if clntMD5B64[0] == "" {
-			return ErrInvalidDigest
-		}
-		md5Sum, err := base64.StdEncoding.Strict().DecodeString(clntMD5B64[0])
-		if err != nil {
-			return ErrInvalidDigest
-		}
-		if !bytes.Equal(md5Sum, getMD5Sum(payload)) {
-			return ErrBadDigest
-		}
-	}
-
-	if skipContentSha256Cksum(r) {
-		return ErrNone
-	}
-
-	// Verify that X-Amz-Content-Sha256 Header == sha256(payload)
-	// If X-Amz-Content-Sha256 header is not sent then we don't calculate/verify sha256(payload)
-	sumHex, ok := r.Header["X-Amz-Content-Sha256"]
-	if isRequestPresignedSignatureV4(r) {
-		sumHex, ok = r.URL.Query()["X-Amz-Content-Sha256"]
-	}
-	if ok {
-		if sumHex[0] == "" {
-			return ErrContentSHA256Mismatch
-		}
-		sum, err := hex.DecodeString(sumHex[0])
-		if err != nil {
-			return ErrContentSHA256Mismatch
-		}
-		if !bytes.Equal(sum, getSHA256Sum(payload)) {
-			return ErrContentSHA256Mismatch
-		}
+		return toAPIErrorCode(err)
 	}
+	r.Body = ioutil.NopCloser(reader)
 	return ErrNone
 }
 

cmd/auth-handler_test.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -361,8 +362,6 @@ func TestIsReqAuthenticated(t *testing.T) {
 		req     *http.Request
 		s3Error APIErrorCode
 	}{
-		// When request is nil, internal error is returned.
-		{nil, ErrInternalError},
 		// When request is unsigned, access denied is returned.
 		{mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrAccessDenied},
 		// Empty Content-Md5 header.
@@ -376,9 +375,11 @@ func TestIsReqAuthenticated(t *testing.T) {
 	}
 
 	// Validates all testcases.
-	for _, testCase := range testCases {
+	for i, testCase := range testCases {
 		if s3Error := isReqAuthenticated(testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
-			t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
+			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(err) != testCase.s3Error {
+				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %d)", i, testCase.s3Error, s3Error, toAPIErrorCode(err))
+			}
 		}
 	}
 }

cmd/auth-rpc-client.go

@@ -1,310 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bufio"
-	"crypto/tls"
-	"crypto/x509"
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"net/rpc"
-	"strings"
-	"sync"
-	"time"
-)
-
-// Attempt to retry only this many number of times before
-// giving up on the remote RPC entirely.
-const globalAuthRPCRetryThreshold = 1
-
-// authConfig requires to make new AuthRPCClient.
-type authConfig struct {
-	accessKey        string // Access key (like username) for authentication.
-	secretKey        string // Secret key (like Password) for authentication.
-	serverAddr       string // RPC server address.
-	serviceEndpoint  string // Endpoint on the server to make any RPC call.
-	secureConn       bool   // Make TLS connection to RPC server or not.
-	serviceName      string // Service name of auth server.
-	disableReconnect bool   // Disable reconnect on failure or not.
-
-	/// Retry configurable values.
-
-	// Each retry unit multiplicative, measured in time.Duration.
-	// This is the basic unit used for calculating backoffs.
-	retryUnit time.Duration
-	// Maximum retry duration i.e A caller would wait no more than this
-	// duration to continue their loop.
-	retryCap time.Duration
-
-	// Maximum retries an call authRPC client would do for a failed
-	// RPC call.
-	retryAttemptThreshold int
-}
-
-// AuthRPCClient is a authenticated RPC client which does authentication before doing Call().
-type AuthRPCClient struct {
-	sync.RWMutex             // Mutex to lock this object.
-	rpcClient    *rpc.Client // RPC client to make any RPC call.
-	config       authConfig  // Authentication configuration information.
-	authToken    string      // Authentication token.
-	version      semVersion  // RPC version.
-}
-
-// newAuthRPCClient - returns a JWT based authenticated (go) rpc client, which does automatic reconnect.
-func newAuthRPCClient(config authConfig) *AuthRPCClient {
-	// Check if retry params are set properly if not default them.
-	emptyDuration := time.Duration(int64(0))
-	if config.retryUnit == emptyDuration {
-		config.retryUnit = defaultRetryUnit
-	}
-	if config.retryCap == emptyDuration {
-		config.retryCap = defaultRetryCap
-	}
-	if config.retryAttemptThreshold == 0 {
-		config.retryAttemptThreshold = globalAuthRPCRetryThreshold
-	}
-
-	return &AuthRPCClient{
-		config:  config,
-		version: globalRPCAPIVersion,
-	}
-}
-
-// Login a JWT based authentication is performed with rpc server.
-func (authClient *AuthRPCClient) Login() (err error) {
-	// Login should be attempted one at a time.
-	//
-	// The reason for large region lock here is
-	// to avoid two simultaneous login attempts
-	// racing over each other.
-	//
-	// #1 Login() gets the lock proceeds to login.
-	// #2 Login() waits for the unlock to happen
-	//    after login in #1.
-	// #1 Successfully completes login saves the
-	//    newly acquired token.
-	// #2 Successfully gets the lock and proceeds,
-	//    but since we have acquired the token
-	//    already the call quickly returns.
-	authClient.Lock()
-	defer authClient.Unlock()
-
-	// Attempt to login if not logged in already.
-	if authClient.authToken == "" {
-		var authToken string
-		authToken, err = authenticateNode(authClient.config.accessKey, authClient.config.secretKey)
-		if err != nil {
-			return err
-		}
-
-		// Login to authenticate your token.
-		var (
-			loginMethod = authClient.config.serviceName + loginMethodName
-			loginArgs   = LoginRPCArgs{
-				AuthToken:   authToken,
-				Version:     globalRPCAPIVersion,
-				RequestTime: UTCNow(),
-			}
-		)
-
-		// Re-dial after we have disconnected or if its a fresh run.
-		var rpcClient *rpc.Client
-		rpcClient, err = rpcDial(authClient.config.serverAddr, authClient.config.serviceEndpoint, authClient.config.secureConn)
-		if err != nil {
-			return err
-		}
-
-		if err = rpcClient.Call(loginMethod, &loginArgs, &LoginRPCReply{}); err != nil {
-			// gob doesn't provide any typed errors for us to reflect
-			// upon, this is the only way to return proper error.
-			if strings.Contains(err.Error(), "gob: wrong type") {
-				return errRPCAPIVersionUnsupported
-			}
-			return err
-		}
-
-		// Initialize rpc client and auth token after a successful login.
-		authClient.authToken = authToken
-		authClient.rpcClient = rpcClient
-	}
-
-	return nil
-}
-
-// call makes a RPC call after logs into the server.
-func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
-	SetAuthToken(authToken string)
-	SetRPCAPIVersion(version semVersion)
-}, reply interface{}) (err error) {
-	if err = authClient.Login(); err != nil {
-		return err
-	} // On successful login, execute RPC call.
-
-	// Set token before the rpc call.
-	authClient.RLock()
-	defer authClient.RUnlock()
-	args.SetAuthToken(authClient.authToken)
-	args.SetRPCAPIVersion(authClient.version)
-
-	// Do an RPC call.
-	return authClient.rpcClient.Call(serviceMethod, args, reply)
-}
-
-// Call executes RPC call till success or globalAuthRPCRetryThreshold on ErrShutdown.
-func (authClient *AuthRPCClient) Call(serviceMethod string, args interface {
-	SetAuthToken(authToken string)
-	SetRPCAPIVersion(version semVersion)
-}, reply interface{}) (err error) {
-
-	// Done channel is used to close any lingering retry routine, as soon
-	// as this function returns.
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
-	for i := range newRetryTimer(authClient.config.retryUnit, authClient.config.retryCap, doneCh) {
-		if err = authClient.call(serviceMethod, args, reply); err == rpc.ErrShutdown {
-			// As connection at server side is closed, close the rpc client.
-			authClient.Close()
-
-			// Retry if reconnect is not disabled.
-			if !authClient.config.disableReconnect {
-				// Retry until threshold reaches.
-				if i < authClient.config.retryAttemptThreshold {
-					continue
-				}
-			}
-		}
-		// gob doesn't provide any typed errors for us to reflect
-		// upon, this is the only way to return proper error.
-		if err != nil && strings.Contains(err.Error(), "gob: wrong type") {
-			err = errRPCAPIVersionUnsupported
-		}
-		break
-	}
-	return err
-}
-
-// Close closes underlying RPC Client.
-func (authClient *AuthRPCClient) Close() error {
-	authClient.Lock()
-	defer authClient.Unlock()
-
-	if authClient.rpcClient == nil {
-		return nil
-	}
-
-	authClient.authToken = ""
-	return authClient.rpcClient.Close()
-}
-
-// ServerAddr returns the serverAddr (network address) of the connection.
-func (authClient *AuthRPCClient) ServerAddr() string {
-	return authClient.config.serverAddr
-}
-
-// ServiceEndpoint returns the RPC service endpoint of the connection.
-func (authClient *AuthRPCClient) ServiceEndpoint() string {
-	return authClient.config.serviceEndpoint
-}
-
-// default Dial timeout for RPC connections.
-const defaultDialTimeout = 3 * time.Second
-
-// Connect success message required from rpc server.
-const connectSuccessMessage = "200 Connected to Go RPC"
-
-// dial tries to establish a connection to serverAddr in a safe manner.
-// If there is a valid rpc.Cliemt, it returns that else creates a new one.
-func rpcDial(serverAddr, serviceEndpoint string, secureConn bool) (netRPCClient *rpc.Client, err error) {
-	if serverAddr == "" || serviceEndpoint == "" {
-		return nil, errInvalidArgument
-	}
-	d := &net.Dialer{
-		Timeout: defaultDialTimeout,
-	}
-	var conn net.Conn
-	if secureConn {
-		var hostname string
-		if hostname, _, err = net.SplitHostPort(serverAddr); err != nil {
-			return nil, &net.OpError{
-				Op:   "dial-http",
-				Net:  serverAddr + serviceEndpoint,
-				Addr: nil,
-				Err:  fmt.Errorf("Unable to parse server address <%s>: %s", serverAddr, err),
-			}
-		}
-		// ServerName in tls.Config needs to be specified to support SNI certificates.
-		conn, err = tls.DialWithDialer(d, "tcp", serverAddr, &tls.Config{
-			ServerName: hostname,
-			RootCAs:    globalRootCAs,
-		})
-	} else {
-		conn, err = d.Dial("tcp", serverAddr)
-	}
-
-	if err != nil {
-		// Print RPC connection errors that are worthy to display in log.
-		switch err.(type) {
-		case x509.HostnameError:
-			errorIf(err, "Unable to establish secure connection to %s", serverAddr)
-		}
-
-		return nil, &net.OpError{
-			Op:   "dial-http",
-			Net:  serverAddr + serviceEndpoint,
-			Addr: nil,
-			Err:  err,
-		}
-	}
-
-	// Check for network errors writing over the dialed conn.
-	if _, err = io.WriteString(conn, "CONNECT "+serviceEndpoint+" HTTP/1.0\n\n"); err != nil {
-		conn.Close()
-		return nil, &net.OpError{
-			Op:   "dial-http",
-			Net:  serverAddr + serviceEndpoint,
-			Addr: nil,
-			Err:  err,
-		}
-	}
-
-	// Attempt to read the HTTP response for the HTTP method CONNECT, upon
-	// success return the RPC connection instance.
-	resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{
-		Method: http.MethodConnect,
-	})
-	if err != nil {
-		conn.Close()
-		return nil, &net.OpError{
-			Op:   "dial-http",
-			Net:  serverAddr + serviceEndpoint,
-			Addr: nil,
-			Err:  err,
-		}
-	}
-	if resp.Status != connectSuccessMessage {
-		conn.Close()
-		return nil, errors.New("unexpected HTTP response: " + resp.Status)
-	}
-
-	// Initialize rpc client.
-	return rpc.NewClient(conn), nil
-}

cmd/auth-rpc-client_test.go

@@ -1,138 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/x509"
-	"os"
-	"path"
-	"testing"
-)
-
-// Tests authorized RPC client.
-func TestAuthRPCClient(t *testing.T) {
-	// reset globals.
-	// this is to make sure that the tests are not affected by modified globals.
-	resetTestGlobals()
-
-	authCfg := authConfig{
-		accessKey:       "123",
-		secretKey:       "123",
-		serverAddr:      "localhost:9000",
-		serviceEndpoint: "/rpc/disk",
-		secureConn:      false,
-		serviceName:     "MyPackage",
-	}
-	authRPC := newAuthRPCClient(authCfg)
-	if authRPC.ServerAddr() != authCfg.serverAddr {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServerAddr(), authCfg.serverAddr)
-	}
-	if authRPC.ServiceEndpoint() != authCfg.serviceEndpoint {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServiceEndpoint(), authCfg.serviceEndpoint)
-	}
-	authCfg = authConfig{
-		accessKey:   "123",
-		secretKey:   "123",
-		secureConn:  false,
-		serviceName: "MyPackage",
-	}
-	authRPC = newAuthRPCClient(authCfg)
-	if authRPC.ServerAddr() != authCfg.serverAddr {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServerAddr(), authCfg.serverAddr)
-	}
-	if authRPC.ServiceEndpoint() != authCfg.serviceEndpoint {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServiceEndpoint(), authCfg.serviceEndpoint)
-	}
-}
-
-// Test rpc dial test.
-func TestRPCDial(t *testing.T) {
-	prevRootCAs := globalRootCAs
-	defer func() {
-		globalRootCAs = prevRootCAs
-	}()
-
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	testServer := StartTestServer(t, "")
-	defer testServer.Stop()
-
-	cert, key, err := generateTLSCertKey("127.0.0.1")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Set global root CAs.
-	globalRootCAs = x509.NewCertPool()
-	globalRootCAs.AppendCertsFromPEM(cert)
-
-	testServerTLS := StartTestTLSServer(t, "", cert, key)
-	defer testServerTLS.Stop()
-
-	adminEndpoint := path.Join(minioReservedBucketPath, adminPath)
-	testCases := []struct {
-		serverAddr     string
-		serverEndpoint string
-		success        bool
-		secure         bool
-	}{
-		// Empty server addr should fail.
-		{
-			serverAddr:     "",
-			serverEndpoint: adminEndpoint,
-			success:        false,
-		},
-		// Unexpected server addr should fail.
-		{
-			serverAddr:     "example.com",
-			serverEndpoint: adminEndpoint,
-			success:        false,
-		},
-		// Server addr connects but fails for CONNECT call.
-		{
-			serverAddr:     "example.com:80",
-			serverEndpoint: "/",
-			success:        false,
-		},
-		// Successful connecting to insecure RPC server.
-		{
-			serverAddr:     testServer.Server.Listener.Addr().String(),
-			serverEndpoint: adminEndpoint,
-			success:        true,
-		},
-		// Successful connecting to secure RPC server.
-		{
-			serverAddr:     testServerTLS.Server.Listener.Addr().String(),
-			serverEndpoint: adminEndpoint,
-			success:        true,
-			secure:         true,
-		},
-	}
-	for i, testCase := range testCases {
-		_, err = rpcDial(testCase.serverAddr, testCase.serverEndpoint, testCase.secure)
-		if err != nil && testCase.success {
-			t.Errorf("Test %d: Expected success but found failure instead %s", i+1, err)
-		}
-		if err == nil && !testCase.success {
-			t.Errorf("Test %d: Expected failure but found success instead", i+1)
-		}
-	}
-}

cmd/auth-rpc-server_test.go

@@ -1,88 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"os"
-	"testing"
-	"time"
-)
-
-func TestLogin(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatalf("Failed to create test config - %v", err)
-	}
-	defer os.RemoveAll(rootPath)
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ls := AuthRPCServer{}
-	testCases := []struct {
-		args        LoginRPCArgs
-		skewTime    time.Duration
-		expectedErr error
-	}{
-		// Valid case.
-		{
-			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   globalRPCAPIVersion,
-			},
-			skewTime:    0,
-			expectedErr: nil,
-		},
-		// Valid username, password and request time, not version.
-		{
-			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   semVersion{1, 0, 0},
-			},
-			skewTime:    0,
-			expectedErr: errRPCAPIVersionUnsupported,
-		},
-		// Valid username, password and version, not request time
-		{
-			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   globalRPCAPIVersion,
-			},
-			skewTime:    20 * time.Minute,
-			expectedErr: errServerTimeMismatch,
-		},
-		// Invalid token, fails with authentication error
-		{
-			args: LoginRPCArgs{
-				AuthToken: "",
-				Version:   globalRPCAPIVersion,
-			},
-			skewTime:    0,
-			expectedErr: errAuthentication,
-		},
-	}
-	for i, test := range testCases {
-		reply := LoginRPCReply{}
-		test.args.RequestTime = UTCNow().Add(test.skewTime)
-		err := ls.Login(&test.args, &reply)
-		if err != test.expectedErr {
-			t.Errorf("Test %d: Expected error %v but received %v",
-				i+1, test.expectedErr, err)
-		}
-	}
-}

cmd/bool-flag.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,11 +21,11 @@ import (
 	"fmt"
 )
 
-// BrowserFlag - wrapper bool type.
-type BrowserFlag bool
+// BoolFlag - wrapper bool type.
+type BoolFlag bool
 
-// String - returns string of BrowserFlag.
-func (bf BrowserFlag) String() string {
+// String - returns string of BoolFlag.
+func (bf BoolFlag) String() string {
 	if bf {
 		return "on"
 	}
@@ -33,20 +33,20 @@ func (bf BrowserFlag) String() string {
 	return "off"
 }
 
-// MarshalJSON - converts BrowserFlag into JSON data.
-func (bf BrowserFlag) MarshalJSON() ([]byte, error) {
+// MarshalJSON - converts BoolFlag into JSON data.
+func (bf BoolFlag) MarshalJSON() ([]byte, error) {
 	return json.Marshal(bf.String())
 }
 
-// UnmarshalJSON - parses given data into BrowserFlag.
-func (bf *BrowserFlag) UnmarshalJSON(data []byte) (err error) {
+// UnmarshalJSON - parses given data into BoolFlag.
+func (bf *BoolFlag) UnmarshalJSON(data []byte) (err error) {
 	var s string
 	if err = json.Unmarshal(data, &s); err == nil {
-		b := BrowserFlag(true)
+		b := BoolFlag(true)
 		if s == "" {
 			// Empty string is treated as valid.
 			*bf = b
-		} else if b, err = ParseBrowserFlag(s); err == nil {
+		} else if b, err = ParseBoolFlag(s); err == nil {
 			*bf = b
 		}
 	}
@@ -54,14 +54,14 @@ func (bf *BrowserFlag) UnmarshalJSON(data []byte) (err error) {
 	return err
 }
 
-// ParseBrowserFlag - parses string into BrowserFlag.
-func ParseBrowserFlag(s string) (bf BrowserFlag, err error) {
+// ParseBoolFlag - parses string into BoolFlag.
+func ParseBoolFlag(s string) (bf BoolFlag, err error) {
 	if s == "on" {
 		bf = true
 	} else if s == "off" {
 		bf = false
 	} else {
-		err = fmt.Errorf("invalid value ‘%s’ for BrowserFlag", s)
+		err = fmt.Errorf("invalid value ‘%s’ for BoolFlag", s)
 	}
 
 	return bf, err

cmd/bool-flag_test.go

@@ -21,17 +21,17 @@ import (
 	"testing"
 )
 
-// Test BrowserFlag.String()
-func TestBrowserFlagString(t *testing.T) {
-	var bf BrowserFlag
+// Test BoolFlag.String()
+func TestBoolFlagString(t *testing.T) {
+	var bf BoolFlag
 
 	testCases := []struct {
-		flag           BrowserFlag
+		flag           BoolFlag
 		expectedResult string
 	}{
 		{bf, "off"},
-		{BrowserFlag(true), "on"},
-		{BrowserFlag(false), "off"},
+		{BoolFlag(true), "on"},
+		{BoolFlag(false), "off"},
 	}
 
 	for _, testCase := range testCases {
@@ -42,17 +42,17 @@ func TestBrowserFlagString(t *testing.T) {
 	}
 }
 
-// Test BrowserFlag.MarshalJSON()
-func TestBrowserFlagMarshalJSON(t *testing.T) {
-	var bf BrowserFlag
+// Test BoolFlag.MarshalJSON()
+func TestBoolFlagMarshalJSON(t *testing.T) {
+	var bf BoolFlag
 
 	testCases := []struct {
-		flag           BrowserFlag
+		flag           BoolFlag
 		expectedResult string
 	}{
 		{bf, `"off"`},
-		{BrowserFlag(true), `"on"`},
-		{BrowserFlag(false), `"off"`},
+		{BoolFlag(true), `"on"`},
+		{BoolFlag(false), `"off"`},
 	}
 
 	for _, testCase := range testCases {
@@ -63,27 +63,27 @@ func TestBrowserFlagMarshalJSON(t *testing.T) {
 	}
 }
 
-// Test BrowserFlag.UnmarshalJSON()
-func TestBrowserFlagUnmarshalJSON(t *testing.T) {
+// Test BoolFlag.UnmarshalJSON()
+func TestBoolFlagUnmarshalJSON(t *testing.T) {
 	testCases := []struct {
 		data           []byte
-		expectedResult BrowserFlag
+		expectedResult BoolFlag
 		expectedErr    error
 	}{
-		{[]byte(`{}`), BrowserFlag(false), errors.New("json: cannot unmarshal object into Go value of type string")},
-		{[]byte(`["on"]`), BrowserFlag(false), errors.New("json: cannot unmarshal array into Go value of type string")},
-		{[]byte(`"junk"`), BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
-		{[]byte(`"true"`), BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
-		{[]byte(`"false"`), BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
-		{[]byte(`"ON"`), BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
-		{[]byte(`"OFF"`), BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
-		{[]byte(`""`), BrowserFlag(true), nil},
-		{[]byte(`"on"`), BrowserFlag(true), nil},
-		{[]byte(`"off"`), BrowserFlag(false), nil},
+		{[]byte(`{}`), BoolFlag(false), errors.New("json: cannot unmarshal object into Go value of type string")},
+		{[]byte(`["on"]`), BoolFlag(false), errors.New("json: cannot unmarshal array into Go value of type string")},
+		{[]byte(`"junk"`), BoolFlag(false), errors.New("invalid value ‘junk’ for BoolFlag")},
+		{[]byte(`"true"`), BoolFlag(false), errors.New("invalid value ‘true’ for BoolFlag")},
+		{[]byte(`"false"`), BoolFlag(false), errors.New("invalid value ‘false’ for BoolFlag")},
+		{[]byte(`"ON"`), BoolFlag(false), errors.New("invalid value ‘ON’ for BoolFlag")},
+		{[]byte(`"OFF"`), BoolFlag(false), errors.New("invalid value ‘OFF’ for BoolFlag")},
+		{[]byte(`""`), BoolFlag(true), nil},
+		{[]byte(`"on"`), BoolFlag(true), nil},
+		{[]byte(`"off"`), BoolFlag(false), nil},
 	}
 
 	for _, testCase := range testCases {
-		var flag BrowserFlag
+		var flag BoolFlag
 		err := (&flag).UnmarshalJSON(testCase.data)
 		if testCase.expectedErr == nil {
 			if err != nil {
@@ -101,25 +101,25 @@ func TestBrowserFlagUnmarshalJSON(t *testing.T) {
 	}
 }
 
-// Test ParseBrowserFlag()
-func TestParseBrowserFlag(t *testing.T) {
+// Test ParseBoolFlag()
+func TestParseBoolFlag(t *testing.T) {
 	testCases := []struct {
 		flagStr        string
-		expectedResult BrowserFlag
+		expectedResult BoolFlag
 		expectedErr    error
 	}{
-		{"", BrowserFlag(false), errors.New("invalid value ‘’ for BrowserFlag")},
-		{"junk", BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
-		{"true", BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
-		{"false", BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
-		{"ON", BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
-		{"OFF", BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
-		{"on", BrowserFlag(true), nil},
-		{"off", BrowserFlag(false), nil},
+		{"", BoolFlag(false), errors.New("invalid value ‘’ for BoolFlag")},
+		{"junk", BoolFlag(false), errors.New("invalid value ‘junk’ for BoolFlag")},
+		{"true", BoolFlag(false), errors.New("invalid value ‘true’ for BoolFlag")},
+		{"false", BoolFlag(false), errors.New("invalid value ‘false’ for BoolFlag")},
+		{"ON", BoolFlag(false), errors.New("invalid value ‘ON’ for BoolFlag")},
+		{"OFF", BoolFlag(false), errors.New("invalid value ‘OFF’ for BoolFlag")},
+		{"on", BoolFlag(true), nil},
+		{"off", BoolFlag(false), nil},
 	}
 
 	for _, testCase := range testCases {
-		bf, err := ParseBrowserFlag(testCase.flagStr)
+		bf, err := ParseBoolFlag(testCase.flagStr)
 		if testCase.expectedErr == nil {
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)

cmd/browser-peer-rpc.go

@@ -1,140 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"fmt"
-	"path"
-	"sync"
-	"time"
-
-	"github.com/minio/minio/pkg/auth"
-)
-
-// SetAuthPeerArgs - Arguments collection for SetAuth RPC call
-type SetAuthPeerArgs struct {
-	// For Auth
-	AuthRPCArgs
-
-	// New credentials that receiving peer should update to.
-	Creds auth.Credentials
-}
-
-// SetAuthPeer - Update to new credentials sent from a peer Minio
-// server. Since credentials are already validated on the sending
-// peer, here we just persist to file and update in-memory config. All
-// subsequently running isAuthTokenValid() calls will fail, and clients
-// will be forced to re-establish connections. Connections will be
-// re-established only when the sending client has also updated its
-// credentials.
-func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthRPCReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	if !args.Creds.IsValid() {
-		return fmt.Errorf("Invalid credential passed")
-	}
-
-	// Acquire lock before updating global configuration.
-	globalServerConfigMu.Lock()
-	defer globalServerConfigMu.Unlock()
-
-	// Update credentials in memory
-	prevCred := globalServerConfig.SetCredential(args.Creds)
-
-	// Save credentials to config file
-	if err := globalServerConfig.Save(); err != nil {
-		// Save the current creds when failed to update.
-		globalServerConfig.SetCredential(prevCred)
-
-		errorIf(err, "Unable to update the config with new credentials sent from browser RPC.")
-		return err
-	}
-
-	return nil
-}
-
-// Sends SetAuthPeer RPCs to all peers in the Minio cluster
-func updateCredsOnPeers(creds auth.Credentials) map[string]error {
-	peers := GetRemotePeers(globalEndpoints)
-
-	// Array of errors for each peer
-	errs := make([]error, len(peers))
-	var wg sync.WaitGroup
-
-	serverCred := globalServerConfig.GetCredential()
-	// Launch go routines to send request to each peer in parallel.
-	for ix := range peers {
-		wg.Add(1)
-		go func(ix int) {
-			defer wg.Done()
-
-			// Exclude self to avoid race with
-			// invalidating the RPC token.
-			if peers[ix] == globalMinioAddr {
-				errs[ix] = nil
-				return
-			}
-
-			// Initialize client
-			client := newAuthRPCClient(authConfig{
-				accessKey:       serverCred.AccessKey,
-				secretKey:       serverCred.SecretKey,
-				serverAddr:      peers[ix],
-				secureConn:      globalIsSSL,
-				serviceEndpoint: path.Join(minioReservedBucketPath, browserPeerPath),
-				serviceName:     "BrowserPeer",
-			})
-
-			// Construct RPC call arguments.
-			args := SetAuthPeerArgs{Creds: creds}
-
-			// Make RPC call - we only care about error
-			// response and not the reply.
-			err := client.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{})
-
-			// We try a bit hard (3 attempts with 1 second delay)
-			// to set creds on peers in case of failure.
-			if err != nil {
-				for i := 0; i < 2; i++ {
-					time.Sleep(1 * time.Second) // 1 second delay.
-					err = client.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{})
-					if err == nil {
-						break
-					}
-				}
-			}
-
-			// Send result down the channel
-			errs[ix] = err
-		}(ix)
-	}
-
-	// Wait for requests to complete.
-	wg.Wait()
-
-	// Put errors into map.
-	errsMap := make(map[string]error)
-	for i, err := range errs {
-		if err != nil {
-			errsMap[peers[i]] = err
-		}
-	}
-
-	return errsMap
-}

cmd/browser-peer-rpc_test.go

@@ -1,116 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"path"
-	"testing"
-
-	"github.com/minio/minio/pkg/auth"
-)
-
-// API suite container common to both FS and XL.
-type TestRPCBrowserPeerSuite struct {
-	serverType   string
-	testServer   TestServer
-	testAuthConf authConfig
-}
-
-// Setting up the test suite and starting the Test server.
-func (s *TestRPCBrowserPeerSuite) SetUpSuite(t *testing.T) {
-	s.testServer = StartTestBrowserPeerRPCServer(t, s.serverType)
-	s.testAuthConf = authConfig{
-		serverAddr:      s.testServer.Server.Listener.Addr().String(),
-		accessKey:       s.testServer.AccessKey,
-		secretKey:       s.testServer.SecretKey,
-		serviceEndpoint: path.Join(minioReservedBucketPath, browserPeerPath),
-		serviceName:     "BrowserPeer",
-	}
-}
-
-// TeatDownSuite - called implicitly by after all tests are run in
-// browser peer rpc suite.
-func (s *TestRPCBrowserPeerSuite) TearDownSuite(t *testing.T) {
-	s.testServer.Stop()
-}
-
-func TestBrowserPeerRPC(t *testing.T) {
-	// setup code
-	s := &TestRPCBrowserPeerSuite{serverType: "XL"}
-	s.SetUpSuite(t)
-
-	// run test
-	s.testBrowserPeerRPC(t)
-
-	// teardown code
-	s.TearDownSuite(t)
-}
-
-// Tests for browser peer rpc.
-func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
-	// Construct RPC call arguments.
-	creds, err := auth.CreateCredentials("abcd1", "abcd1234")
-	if err != nil {
-		t.Fatalf("unable to create credential. %v", err)
-	}
-
-	// Validate for invalid token.
-	args := SetAuthPeerArgs{Creds: creds}
-	rclient := newAuthRPCClient(s.testAuthConf)
-	defer rclient.Close()
-	if err = rclient.Login(); err != nil {
-		t.Fatal(err)
-	}
-	rclient.authToken = "garbage"
-	if err = rclient.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{}); err != nil {
-		if err.Error() != errInvalidToken.Error() {
-			t.Fatal(err)
-		}
-	}
-
-	// Validate for successful Peer update.
-	args = SetAuthPeerArgs{Creds: creds}
-	client := newAuthRPCClient(s.testAuthConf)
-	defer client.Close()
-	err = client.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Validate for failure in login handler with previous credentials.
-	rclient = newAuthRPCClient(s.testAuthConf)
-	defer rclient.Close()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	rclient.authToken = token
-	if err = rclient.Login(); err != nil {
-		if err.Error() != errInvalidAccessKeyID.Error() {
-			t.Fatal(err)
-		}
-	}
-
-	token, err = authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	rclient.authToken = token
-	if err = rclient.Login(); err != nil {
-		t.Fatal(err)
-	}
-}

cmd/browser-rpc-router.go

@@ -1,51 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	router "github.com/gorilla/mux"
-
-	"github.com/minio/minio/pkg/errors"
-)
-
-// Set up an RPC endpoint that receives browser related calls. The
-// original motivation is for propagating credentials change
-// throughout Minio cluster, initiated from a Minio browser session.
-
-const (
-	browserPeerPath = "/browser/setauth"
-)
-
-// The Type exporting methods exposed for RPC calls.
-type browserPeerAPIHandlers struct {
-	AuthRPCServer
-}
-
-// Register RPC router
-func registerBrowserPeerRPCRouter(mux *router.Router) error {
-	bpHandlers := &browserPeerAPIHandlers{AuthRPCServer{}}
-
-	bpRPCServer := newRPCServer()
-	err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)
-	if err != nil {
-		return errors.Trace(err)
-	}
-
-	bpRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
-	bpRouter.Path(browserPeerPath).Handler(bpRPCServer)
-	return nil
-}

cmd/bucket-handlers-listobjects.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // Validate all the ListObjects query arguments, returns an APIErrorCode
@@ -64,13 +65,20 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
+	urlValues := r.URL.Query()
+
 	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
+
+	if errCode != ErrNone {
+		writeErrorResponse(w, errCode, r.URL)
+		return
+	}
 
 	// In ListObjectsV2 'continuation-token' is the marker.
 	marker := token
@@ -92,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
+	// marshaled into S3 compatible XML header.
 	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -134,7 +142,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -158,7 +166,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
+	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/bucket-handlers.go

@@ -20,89 +20,69 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/xml"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/url"
 	"path"
 	"path/filepath"
-	"reflect"
 	"strings"
 	"sync"
 
-	mux "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/pkg/errors"
+	etcd "github.com/coreos/etcd/client"
+
+	"github.com/gorilla/mux"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/sync/errgroup"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
-// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-// Enforces bucket policies for a bucket for a given tatusaction.
-func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, queryParams url.Values) (s3Error APIErrorCode) {
-	// Verify if bucket actually exists
-	objAPI := newObjectLayerFn()
-	if err := checkBucketExist(bucket, objAPI); err != nil {
-		err = errors.Cause(err)
-		switch err.(type) {
-		case BucketNameInvalid:
-			// Return error for invalid bucket name.
-			return ErrInvalidBucketName
-		case BucketNotFound:
-			// For no bucket found we return NoSuchBucket instead.
-			return ErrNoSuchBucket
+// Check if there are buckets on server without corresponding entry in etcd backend and
+// make entries. Here is the general flow
+// - Range over all the available buckets
+// - Check if a bucket has an entry in etcd backend
+// -- If no, make an entry
+// -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
+// -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
+func initFederatorBackend(objLayer ObjectLayer) {
+	b, err := objLayer.ListBuckets(context.Background())
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return
+	}
+
+	g := errgroup.WithNErrs(len(b))
+	for index := range b {
+		index := index
+		g.Go(func() error {
+			r, gerr := globalDNSConfig.Get(b[index].Name)
+			if gerr != nil {
+				if etcd.IsKeyNotFound(gerr) || gerr == dns.ErrNoEntriesFound {
+					return globalDNSConfig.Put(b[index].Name)
+				}
+				return gerr
+			}
+			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				// There is already an entry for this bucket, with all IP addresses different. This indicates a bucket name collision. Log an error and continue.
+				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", b[index].Name, globalDomainIPs.ToSlice())
+			}
+			return nil
+		}, index)
+	}
+
+	for _, err := range g.Wait() {
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			return
 		}
-		errorIf(err, "Unable to read bucket policy.")
-		// Return internal error for any other errors so that we can investigate.
-		return ErrInternalError
 	}
-
-	// Fetch bucket policy, if policy is not set return access denied.
-	p, err := objAPI.GetBucketPolicy(context.Background(), bucket)
-	if err != nil {
-		return ErrAccessDenied
-	}
-	if reflect.DeepEqual(p, emptyBucketPolicy) {
-		return ErrAccessDenied
-	}
-
-	// Construct resource in 'arn:aws:s3:::examplebucket/object' format.
-	arn := bucketARNPrefix + strings.TrimSuffix(strings.TrimPrefix(resource, "/"), "/")
-
-	// Get conditions for policy verification.
-	conditionKeyMap := make(policy.ConditionKeyMap)
-	for queryParam := range queryParams {
-		conditionKeyMap[queryParam] = set.CreateStringSet(queryParams.Get(queryParam))
-	}
-
-	// Add request referer to conditionKeyMap if present.
-	if referer != "" {
-		conditionKeyMap["referer"] = set.CreateStringSet(referer)
-	}
-	// Add request source Ip to conditionKeyMap.
-	conditionKeyMap["ip"] = set.CreateStringSet(sourceIP)
-
-	// Validate action, resource and conditions with current policy statements.
-	if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, p.Statements) {
-		return ErrAccessDenied
-	}
-	return ErrNone
-}
-
-// Check if the action is allowed on the bucket/prefix.
-func isBucketActionAllowed(action, bucket, prefix string, objectAPI ObjectLayer) bool {
-	bp, err := objectAPI.GetBucketPolicy(context.Background(), bucket)
-	if err != nil {
-		return false
-	}
-	if reflect.DeepEqual(bp, emptyBucketPolicy) {
-		return false
-	}
-	resource := bucketARNPrefix + path.Join(bucket, prefix)
-	var conditionKeyMap map[string]set.StringSet
-	// Validate action, resource and conditions with current policy statements.
-	return bucketPolicyEvalStatements(action, resource, conditionKeyMap, bp.Statements)
 }
 
 // GetBucketLocationHandler - GET Bucket location.
@@ -120,12 +100,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	s3Error := checkRequestAuthType(r, bucket, "s3:GetBucketLocation", globalMinioDefaultRegion)
-	if s3Error == ErrInvalidRegion {
-		// Clients like boto3 send getBucketLocation() call signed with region that is configured.
-		s3Error = checkRequestAuthType(r, "", "s3:GetBucketLocation", globalServerConfig.GetRegion())
-	}
-	if s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -179,7 +154,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucketMultipartUploads", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -227,21 +202,38 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	if api.CacheAPI() != nil {
 		listBuckets = api.CacheAPI().ListBuckets
 	}
-	// ListBuckets does not have any bucket action.
-	s3Error := checkRequestAuthType(r, "", "", globalMinioDefaultRegion)
-	if s3Error == ErrInvalidRegion {
-		// Clients like boto3 send listBuckets() call signed with region that is configured.
-		s3Error = checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
-	}
-	if s3Error != ErrNone {
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
-	// Invoke the list buckets.
-	bucketsInfo, err := listBuckets(ctx)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
+	// If etcd, dns federation configured list buckets from etcd.
+	var bucketsInfo []BucketInfo
+	if globalDNSConfig != nil {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil && !etcd.IsKeyNotFound(err) && err != dns.ErrNoEntriesFound {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+		bucketSet := set.NewStringSet()
+		for _, dnsRecord := range dnsBuckets {
+			if bucketSet.Contains(dnsRecord.Key) {
+				continue
+			}
+			bucketsInfo = append(bucketsInfo, BucketInfo{
+				Name:    strings.Trim(dnsRecord.Key, slashSeparator),
+				Created: dnsRecord.CreationDate,
+			})
+			bucketSet.Add(dnsRecord.Key)
+		}
+	} else {
+		// Invoke the list buckets.
+		var err error
+		bucketsInfo, err = listBuckets(ctx)
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	// Generate response.
@@ -265,12 +257,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var authError APIErrorCode
-	if authError = checkRequestAuthType(r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); authError != ErrNone {
+	var s3Error APIErrorCode
+	if s3Error = checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, ""); s3Error != ErrNone {
 		// In the event access is denied, a 200 response should still be returned
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-		if authError != ErrAccessDenied {
-			writeErrorResponse(w, authError, r.URL)
+		if s3Error != ErrAccessDenied {
+			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	}
@@ -290,11 +282,17 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	// Allocate incoming content length bytes.
-	deleteXMLBytes := make([]byte, r.ContentLength)
+	var deleteXMLBytes []byte
+	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
+	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
+		deleteXMLBytes = make([]byte, maxBodySize)
+	} else {
+		deleteXMLBytes = make([]byte, r.ContentLength)
+	}
 
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
-		errorIf(err, "Unable to read HTTP body.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
@@ -302,7 +300,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Unmarshal list of keys to be deleted.
 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
-		errorIf(err, "Unable to unmarshal delete objects request XML.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, ErrMalformedXML, r.URL)
 		return
 	}
@@ -325,7 +323,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			defer wg.Done()
 			// If the request is denied access, each item
 			// should be marked as 'AccessDenied'
-			if authError == ErrAccessDenied {
+			if s3Error == ErrAccessDenied {
 				dErrs[i] = PrefixAccessDenied{
 					Bucket: bucket,
 					Object: obj.ObjectName,
@@ -354,7 +352,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			deletedObjects = append(deletedObjects, object)
 			continue
 		}
-		if _, ok := errors.Cause(err).(ObjectNotFound); ok {
+		if _, ok := err.(ObjectNotFound); ok {
 			// If the object is not found it should be
 			// accounted as deleted as per S3 spec.
 			deletedObjects = append(deletedObjects, object)
@@ -410,16 +408,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// PutBucket does not have any bucket action.
-	s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
-	if s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
@@ -434,12 +430,33 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	if globalDNSConfig != nil {
+		if _, err := globalDNSConfig.Get(bucket); err != nil {
+			if etcd.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
+				// Proceed to creating a bucket.
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
+				if err = globalDNSConfig.Put(bucket); err != nil {
+					objectAPI.DeleteBucket(ctx, bucket)
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
+
+				// Make sure to add Location information here only for bucket
+				w.Header().Set("Location", getObjectLocation(r, globalDomainName, bucket, ""))
+
+				writeSuccessResponseHeadersOnly(w)
+				return
+			}
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+
+		}
+		writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL)
 		return
 	}
-	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
@@ -490,7 +507,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// be loaded in memory, the remaining being put in temporary files.
 	reader, err := r.MultipartReader()
 	if err != nil {
-		errorIf(err, "Unable to initialize multipart reader.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
 		return
 	}
@@ -498,7 +515,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Read multipart data and save in memory and in the disk if needed
 	form, err := reader.ReadForm(maxFormMemory)
 	if err != nil {
-		errorIf(err, "Unable to initialize multipart reader.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
 		return
 	}
@@ -507,9 +524,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	defer form.RemoveAll()
 
 	// Extract all form fields
-	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(form)
+	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
 	if err != nil {
-		errorIf(err, "Unable to parse form values.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
 		return
 	}
@@ -584,16 +601,15 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	// Extract metadata to be saved from received Form.
-	metadata, err := extractMetadataFromHeader(formValues)
+	metadata, err := extractMetadataFromHeader(ctx, formValues)
 	if err != nil {
-		errorIf(err, "found invalid http request header")
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
 
 	hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
 	if err != nil {
-		errorIf(err, "Unable to initialize hashReader.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -690,10 +706,11 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponseHeadersOnly(w, s3Error)
 		return
 	}
+
 	getBucketInfo := objectAPI.GetBucketInfo
 	if api.CacheAPI() != nil {
 		getBucketInfo = api.CacheAPI().GetBucketInfo
@@ -710,20 +727,20 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "DeleteBucket")
 
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
 
-	// DeleteBucket does not have any bucket action.
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
 	deleteBucket := objectAPI.DeleteBucket
 	if api.CacheAPI() != nil {
 		deleteBucket = api.CacheAPI().DeleteBucket
@@ -734,14 +751,20 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	// Notify all peers (including self) to update in-memory state
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
-		errorIf(err, "unable to update policy change in remote peer %v", addr)
+	globalNotificationSys.RemoveNotification(bucket)
+	globalPolicySys.Remove(bucket)
+	for nerr := range globalNotificationSys.DeleteBucket(bucket) {
+		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
+		logger.LogIf(ctx, nerr.Err)
 	}
 
-	globalNotificationSys.RemoveNotification(bucket)
-	for addr, err := range globalNotificationSys.DeleteBucket(bucket) {
-		errorIf(err, "unable to delete bucket in remote peer %v", addr)
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			// Deleting DNS entry failed, attempt to create the bucket again.
+			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	// Write success response.

cmd/bucket-handlers_test.go

@@ -155,7 +155,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getReadOnlyBucketStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestGetBucketLocationHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyBucketStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestGetBucketLocationHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -260,7 +260,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getReadOnlyBucketStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestHeadBucketHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyBucketStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestHeadBucketHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -313,7 +313,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			maxUploads:         "0",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusBadRequest,
+			expectedRespStatus: http.StatusNotFound,
 			shouldPass:         false,
 		},
 		// Test case - 2.
@@ -494,7 +494,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyBucketStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestListMultipartUploadsHandler", bucketName, "", instanceType, apiRouter, anonReq, getWriteOnlyBucketStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestListMultipartUploadsHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -592,7 +592,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "ListBucketsHandler", "", "", instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "ListBucketsHandler", "", "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy("*"))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -793,31 +793,3 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	// `ExecObjectLayerAPINilTest` manages the operation.
 	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
 }
-
-func TestIsBucketActionAllowed(t *testing.T) {
-	ExecObjectLayerAPITest(t, testIsBucketActionAllowedHandler, []string{"BucketLocation"})
-}
-
-func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-
-	testCases := []struct {
-		// input.
-		action              string
-		bucket              string
-		prefix              string
-		isGlobalPoliciesNil bool
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		{"s3:GetBucketLocation", "mybucket", "abc", true, false},
-		{"s3:ListObject", "mybucket", "abc", false, false},
-	}
-	for i, testCase := range testCases {
-		isAllowed := isBucketActionAllowed(testCase.action, testCase.bucket, testCase.prefix, obj)
-		if isAllowed != testCase.shouldPass {
-			t.Errorf("Case %d: Expected the response status to be `%t`, but instead found `%t`", i+1, testCase.shouldPass, isAllowed)
-		}
-
-	}
-}

cmd/bucket-notification-handlers.go

@@ -23,10 +23,11 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	xerrors "github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
 	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/policy"
 )
 
 const (
@@ -43,6 +44,9 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "GetBucketNotification")
 
+	vars := mux.Vars(r)
+	bucketName := vars["bucket"]
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -53,27 +57,23 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
 		return
 	}
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucketName := vars["bucket"]
-
 	_, err := objAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Attempt to successfully load notification config.
-	nConfig, err := readNotificationConfig(objAPI, bucketName)
+	nConfig, err := readNotificationConfig(ctx, objAPI, bucketName)
 	if err != nil {
 		// Ignore errNoSuchNotifications to comply with AWS S3.
-		if xerrors.Cause(err) != errNoSuchNotifications {
-			errorIf(err, "Unable to read notification configuration.")
+		if err != errNoSuchNotifications {
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 			return
 		}
@@ -83,7 +83,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	notificationBytes, err := xml.Marshal(nConfig)
 	if err != nil {
-		errorIf(err, "Unable to marshal notification configuration into XML.", err)
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -106,14 +106,15 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
 		return
 	}
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -145,8 +146,9 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	rulesMap := config.ToRulesMap()
 	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
-	for addr, err := range globalNotificationSys.PutBucketNotification(bucketName, rulesMap) {
-		errorIf(err, "unable to put bucket notification to remote peer %v", addr)
+	for nerr := range globalNotificationSys.PutBucketNotification(bucketName, rulesMap) {
+		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
+		logger.LogIf(ctx, nerr.Err)
 	}
 
 	writeSuccessResponseHeadersOnly(w)
@@ -167,14 +169,15 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
 		return
 	}
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	values := r.URL.Query()
 
 	var prefix string
@@ -217,39 +220,57 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	}
 
 	if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
-		errorIf(err, "Unable to get bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	host := xnet.MustParseHost(r.RemoteAddr)
-	target := target.NewHTTPClientTarget(*host, w)
+	host, err := xnet.ParseHost(r.RemoteAddr)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	target, err := target.NewHTTPClientTarget(*host, w)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
 	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
 
 	if err := globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
-		errorIf(err, "Unable to add httpclient target %v to globalNotificationSys.targetList.", target)
+		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
 	defer globalNotificationSys.RemoveRulesMap(bucketName, rulesMap)
 
-	thisAddr := xnet.MustParseHost(GetLocalPeer(globalEndpoints))
-	if err := SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
-		errorIf(err, "Unable to save HTTP listener %v", target)
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	errors := globalNotificationSys.ListenBucketNotification(bucketName, eventNames, pattern, target.ID(), *thisAddr)
-	for addr, err := range errors {
-		errorIf(err, "unable to call listen bucket notification to remote peer %v", addr)
+	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
+		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
+		logger.LogIf(ctx, err)
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	errCh := globalNotificationSys.ListenBucketNotification(bucketName, eventNames, pattern, target.ID(), *thisAddr)
+	for nerr := range errCh {
+		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
+		logger.LogIf(ctx, nerr.Err)
 	}
 
 	<-target.DoneCh
 
-	if err := RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
-		errorIf(err, "Unable to save HTTP listener %v", target)
+	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
+		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}

cmd/bucket-policy-handlers.go

@@ -19,206 +19,24 @@ package cmd
 import (
 	"encoding/json"
 	"io"
-	"io/ioutil"
-	"net"
 	"net/http"
-	"runtime"
-	"strings"
 
 	humanize "github.com/dustin/go-humanize"
-	mux "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio/pkg/errors"
-	"github.com/minio/minio/pkg/wildcard"
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/policy"
 )
 
-// maximum supported access policy size.
-const maxAccessPolicySize = 20 * humanize.KiByte
+const (
+	// As per AWS S3 specification, 20KiB policy JSON data is allowed.
+	maxBucketPolicySize = 20 * humanize.KiByte
 
-// Verify if a given action is valid for the url path based on the
-// existing bucket access policy.
-func bucketPolicyEvalStatements(action string, resource string, conditions policy.ConditionKeyMap,
-	statements []policy.Statement) bool {
-	for _, statement := range statements {
-		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
-			if statement.Effect == "Allow" {
-				return true
-			}
-			// Do not uncomment kept here for readability.
-			// else statement.Effect == "Deny"
-			return false
-		}
-	}
-	// None match so deny.
-	return false
-}
+	// Policy configuration file.
+	bucketPolicyConfig = "policy.json"
+)
 
-// Verify if action, resource and conditions match input policy statement.
-func bucketPolicyMatchStatement(action string, resource string, conditions policy.ConditionKeyMap,
-	statement policy.Statement) bool {
-	// Verify if action, resource and condition match in given statement.
-	return (bucketPolicyActionMatch(action, statement) &&
-		bucketPolicyResourceMatch(resource, statement) &&
-		bucketPolicyConditionMatch(conditions, statement))
-}
-
-// Verify if given action matches with policy statement.
-func bucketPolicyActionMatch(action string, statement policy.Statement) bool {
-	return !statement.Actions.FuncMatch(actionMatch, action).IsEmpty()
-}
-
-// Match function matches wild cards in 'pattern' for resource.
-func resourceMatch(pattern, resource string) bool {
-	if runtime.GOOS == "windows" {
-		// For windows specifically make sure we are case insensitive.
-		return wildcard.Match(strings.ToLower(pattern), strings.ToLower(resource))
-	}
-	return wildcard.Match(pattern, resource)
-}
-
-// Match function matches wild cards in 'pattern' for action.
-func actionMatch(pattern, action string) bool {
-	return wildcard.MatchSimple(pattern, action)
-}
-
-func refererMatch(pattern, referer string) bool {
-	return wildcard.MatchSimple(pattern, referer)
-}
-
-// isIPInCIDR - checks if a given a IP address is a member of the given subnet.
-func isIPInCIDR(cidr, ip string) bool {
-	// AWS S3 spec says IPs must use standard CIDR notation.
-	// http://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html#example-bucket-policies-use-case-3.
-	_, cidrNet, err := net.ParseCIDR(cidr)
-	if err != nil {
-		return false // If provided CIDR can't be parsed no IP will be in the subnet.
-	}
-	addr := net.ParseIP(ip)
-	return cidrNet.Contains(addr)
-}
-
-// Verify if given resource matches with policy statement.
-func bucketPolicyResourceMatch(resource string, statement policy.Statement) bool {
-	// the resource rule for object could contain "*" wild card.
-	// the requested object can be given access based on the already set bucket policy if
-	// the match is successful.
-	// More info: http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html.
-	return !statement.Resources.FuncMatch(resourceMatch, resource).IsEmpty()
-}
-
-// Verify if given condition matches with policy statement.
-func bucketPolicyConditionMatch(conditions policy.ConditionKeyMap, statement policy.Statement) bool {
-	// Supports following conditions.
-	// - StringEquals
-	// - StringNotEquals
-	// - StringLike
-	// - StringNotLike
-	// - IpAddress
-	// - NotIpAddress
-	//
-	// Supported applicable condition keys for each conditions.
-	// - s3:prefix
-	// - s3:max-keys
-	// - s3:aws-Referer
-	// - s3:aws-SourceIp
-
-	// The following loop evaluates the logical AND of all the
-	// conditions in the statement. Note: we can break out of the
-	// loop if and only if a condition evaluates to false.
-	for condition, conditionKeyVal := range statement.Conditions {
-		prefixConditon := conditionKeyVal["s3:prefix"]
-		maxKeyCondition := conditionKeyVal["s3:max-keys"]
-		if condition == "StringEquals" {
-			// If there is no condition with "s3:prefix" or "s3:max-keys" condition key
-			// then there is nothing to check condition against.
-			if !prefixConditon.IsEmpty() && !prefixConditon.Equals(conditions["prefix"]) {
-				return false
-			}
-			if !maxKeyCondition.IsEmpty() && !maxKeyCondition.Equals(conditions["max-keys"]) {
-				return false
-			}
-		} else if condition == "StringNotEquals" {
-			// If there is no condition with "s3:prefix" or "s3:max-keys" condition key
-			// then there is nothing to check condition against.
-			if !prefixConditon.IsEmpty() && prefixConditon.Equals(conditions["prefix"]) {
-				return false
-			}
-			if !maxKeyCondition.IsEmpty() && maxKeyCondition.Equals(conditions["max-keys"]) {
-				return false
-			}
-		} else if condition == "StringLike" {
-			awsReferers := conditionKeyVal["aws:Referer"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsReferers.IsEmpty() {
-				continue
-			}
-			// wildcard match of referer in statement was not empty.
-			// StringLike has a match, i.e, condition evaluates to true.
-			refererFound := false
-			for referer := range conditions["referer"] {
-				if !awsReferers.FuncMatch(refererMatch, referer).IsEmpty() {
-					refererFound = true
-					break
-				}
-			}
-			// No matching referer found, so the condition is false.
-			if !refererFound {
-				return false
-			}
-		} else if condition == "StringNotLike" {
-			awsReferers := conditionKeyVal["aws:Referer"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsReferers.IsEmpty() {
-				continue
-			}
-			// wildcard match of referer in statement was not empty.
-			// StringNotLike has a match, i.e, condition evaluates to false.
-			for referer := range conditions["referer"] {
-				if !awsReferers.FuncMatch(refererMatch, referer).IsEmpty() {
-					return false
-				}
-			}
-		} else if condition == "IpAddress" {
-			awsIps := conditionKeyVal["aws:SourceIp"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsIps.IsEmpty() {
-				continue
-			}
-			// wildcard match of ip if statement was not empty.
-			// Find a valid ip.
-			ipFound := false
-			for ip := range conditions["ip"] {
-				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
-					ipFound = true
-					break
-				}
-			}
-			if !ipFound {
-				return false
-			}
-		} else if condition == "NotIpAddress" {
-			awsIps := conditionKeyVal["aws:SourceIp"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsIps.IsEmpty() {
-				continue
-			}
-			// wildcard match of ip if statement was not empty.
-			// Find if nothing matches.
-			for ip := range conditions["ip"] {
-				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
-					return false
-				}
-			}
-		}
-	}
-
-	return true
-}
-
-// PutBucketPolicyHandler - PUT Bucket policy
-// -----------------
-// This implementation of the PUT operation uses the policy
-// subresource to add to or replace a policy on a bucket
+// PutBucketPolicyHandler - This HTTP handler stores given bucket policy configuration as per
+// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "PutBucketPolicy")
 
@@ -228,77 +46,61 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	// If Content-Length is unknown or zero, deny the
-	// request. PutBucketPolicy always needs a Content-Length.
-	if r.ContentLength == -1 || r.ContentLength == 0 {
+	// Error out if Content-Length is missing.
+	// PutBucketPolicy always needs Content-Length.
+	if r.ContentLength <= 0 {
 		writeErrorResponse(w, ErrMissingContentLength, r.URL)
 		return
 	}
-	// If Content-Length is greater than maximum allowed policy size.
-	if r.ContentLength > maxAccessPolicySize {
+
+	// Error out if Content-Length is beyond allowed size.
+	if r.ContentLength > maxBucketPolicySize {
 		writeErrorResponse(w, ErrEntityTooLarge, r.URL)
 		return
 	}
 
-	// Read access policy up to maxAccessPolicySize.
-	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
-	// bucket policies are limited to 20KB in size, using a limit reader.
-	policyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		errorIf(err, "Unable to read from client.")
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	// Version in policy must not be empty
+	if bucketPolicy.Version == "" {
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
-	policyInfo := policy.BucketAccessPolicy{}
-	if err = json.Unmarshal(policyBytes, &policyInfo); err != nil {
-		writeErrorResponse(w, ErrInvalidPolicyDocument, r.URL)
-		return
-	}
-	// Parse check bucket policy.
-	if s3Error := checkBucketPolicyResources(bucket, policyInfo); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
 
-	if err = objAPI.SetBucketPolicy(ctx, bucket, policyInfo); err != nil {
-		err = errors.Cause(err)
-		switch err.(type) {
-		case NotImplemented:
-			// Return error for invalid bucket name.
-			writeErrorResponse(w, ErrNotImplemented, r.URL)
-		default:
-			writeErrorResponse(w, ErrMalformedPolicy, r.URL)
-		}
-		return
-	}
-
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
-		errorIf(err, "unable to update policy change in remote peer %v", addr)
+	globalPolicySys.Set(bucket, *bucketPolicy)
+	for nerr := range globalNotificationSys.SetBucketPolicy(bucket, bucketPolicy) {
+		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
+		logger.LogIf(ctx, nerr.Err)
 	}
 
 	// Success.
 	writeSuccessNoContent(w)
 }
 
-// DeleteBucketPolicyHandler - DELETE Bucket policy
-// -----------------
-// This implementation of the DELETE operation uses the policy
-// subresource to add to remove a policy on a bucket.
+// DeleteBucketPolicyHandler - This HTTP handler removes bucket policy configuration.
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "DeleteBucketPolicy")
 
@@ -308,40 +110,36 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	// Delete bucket access policy, by passing an empty policy
-	// struct.
 	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
-		errorIf(err, "unable to update policy change in remote peer %v", addr)
+	globalPolicySys.Remove(bucket)
+	for nerr := range globalNotificationSys.RemoveBucketPolicy(bucket) {
+		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
+		logger.LogIf(ctx, nerr.Err)
 	}
 
 	// Success.
 	writeSuccessNoContent(w)
 }
 
-// GetBucketPolicyHandler - GET Bucket policy
-// -----------------
-// This operation uses the policy
-// subresource to return the policy of a specified bucket.
+// GetBucketPolicyHandler - This HTTP handler returns bucket policy configuration.
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "GetBucketPolicy")
 
@@ -351,35 +149,34 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Read bucket access policy.
-	policy, err := objAPI.GetBucketPolicy(ctx, bucket)
+	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	policyBytes, err := json.Marshal(&policy)
+	policyData, err := json.Marshal(bucketPolicy)
 	if err != nil {
-		errorIf(err, "Unable to marshal bucket policy.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Write to client.
-	w.Write(policyBytes)
+	w.Write(policyData)
 }

cmd/bucket-policy-handlers_test.go

@@ -24,222 +24,72 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
+	"strings"
 	"testing"
 
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/policy/condition"
 )
 
-// Tests validate Bucket policy resource matcher.
-func TestBucketPolicyResourceMatch(t *testing.T) {
-
-	// generates statement with given resource..
-	generateStatement := func(resource string) policy.Statement {
-		statement := policy.Statement{}
-		statement.Resources = set.CreateStringSet([]string{resource}...)
-		return statement
-	}
-
-	// generates resource prefix.
-	generateResource := func(bucketName, objectName string) string {
-		return bucketARNPrefix + bucketName + "/" + objectName
-	}
-
-	testCases := []struct {
-		resourceToMatch       string
-		statement             policy.Statement
-		expectedResourceMatch bool
-	}{
-		// Test case 1-4.
-		// Policy with resource ending with bucket/* allows access to all objects inside the given bucket.
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		// Test case - 5.
-		// Policy with resource ending with bucket/oo* should not allow access to bucket/output.txt.
-		{generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), false},
-		// Test case - 6.
-		// Policy with resource ending with bucket/oo* should allow access to bucket/ootput.txt.
-		{generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), true},
-		// Test case - 7.
-		// Policy with resource ending with bucket/oo* allows access to all sub-dirs starting with "oo" inside given bucket.
-		{generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), true},
-		// Test case - 8.
-		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/Asia/Japan/*")), false},
-		// Test case - 9.
-		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/Asia/Japan/*")), false},
-		// Test case - 10.
-		// Proves that the name space is flat.
-		{generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix,
-			"minio-bucket"+"/*/India/*/Bihar")), true},
-		// Test case - 11.
-		// Proves that the name space is flat.
-		{generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix,
-			"minio-bucket"+"/*/India/*/Bihar/*")), true},
-	}
-	for i, testCase := range testCases {
-		actualResourceMatch := bucketPolicyResourceMatch(testCase.resourceToMatch, testCase.statement)
-		if testCase.expectedResourceMatch != actualResourceMatch {
-			t.Errorf("Test %d: Expected Resource match to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResourceMatch, actualResourceMatch)
-		}
+func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
+			policy.NewResourceSet(policy.NewResource(bucketName, "")),
+			condition.NewFunctions(),
+		)},
 	}
 }
 
-// TestBucketPolicyActionMatch - Test validates whether given action on the
-// bucket/object matches the allowed actions in policy.Statement.
-// This test preserves the allowed actions for all 3 sets of policies, that is read-write,read-only, write-only.
-// The intention of the test is to catch any changes made to allowed action for on eof the above 3 major policy groups mentioned.
-func TestBucketPolicyActionMatch(t *testing.T) {
-	bucketName := getRandomBucketName()
-	objectPrefix := "test-object"
-
-	testCases := []struct {
-		action         string
-		statement      policy.Statement
-		expectedResult bool
-	}{
-		// s3:GetBucketLocation is the action necessary to be present in the bucket policy to allow
-		// fetching of bucket location on an Anonymous/unsigned request.
-
-		//r ead-write bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 1).
-		{"s3:GetBucketLocation", getReadWriteBucketStatement(bucketName, objectPrefix), true},
-		//	write-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 2).
-		{"s3:GetBucketLocation", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
-		//	read-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 3).
-		{"s3:GetBucketLocation", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
-
-		// Any of the Object level access permissions shouldn't allow for GetBucketLocation operation  on an Anonymous/unsigned request (Test cases 4-6).
-		{"s3:GetBucketLocation", getReadWriteObjectStatement(bucketName, objectPrefix), false},
-		{"s3:GetBucketLocation", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-		{"s3:GetBucketLocation", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:ListBucketMultipartUploads is the action necessary to be present in the bucket policy to allow
-		// Listing of multipart uploads in a given bucket for an Anonymous/unsigned request.
-
-		//read-write bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 7).
-		{"s3:ListBucketMultipartUploads", getReadWriteBucketStatement(bucketName, objectPrefix), true},
-		//	write-only bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 8).
-		{"s3:ListBucketMultipartUploads", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
-		// read-only bucket policy is expected to not allow ListBucketMultipartUploads operation on an anonymous request (Test case 9).
-		// the allowed actions in read-only bucket statement are  "s3:GetBucketLocation","s3:ListBucket",
-		// this should not allow for ListBucketMultipartUploads operations.
-		{"s3:ListBucketMultipartUploads", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Any of the object level policy will not allow for s3:ListBucketMultipartUploads (Test cases 10-12).
-		{"s3:ListBucketMultipartUploads", getReadWriteObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucketMultipartUploads", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucketMultipartUploads", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:ListBucket is the action necessary to be present in the bucket policy to allow
-		// listing of all objects inside a given bucket on an Anonymous/unsigned request.
-
-		// Cases for testing ListBucket access for different Bucket level access permissions.
-		// read-only bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 13).
-		{"s3:ListBucket", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
-		// read-write bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 14).
-		{"s3:ListBucket", getReadWriteBucketStatement(bucketName, objectPrefix), true},
-		// write-only  bucket policy is expected to not allow ListBucket operation on an anonymous request (Test case 15).
-		// the allowed actions in write-only  bucket statement are "s3:GetBucketLocation",	"s3:ListBucketMultipartUploads",
-		// this should not allow for ListBucket operations.
-		{"s3:ListBucket", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing ListBucket access for different Object level access permissions (Test cases 16-18).
-		// Any of the Object level access permissions shouldn't allow for ListBucket operation  on an Anonymous/unsigned request.
-		{"s3:ListBucket", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucket", getReadWriteObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucket", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:DeleteObject is the action necessary to be present in the bucket policy to allow
-		// deleting/removal of  objects inside a given bucket for an Anonymous/unsigned request.
-
-		// Cases for testing DeleteObject access for different Bucket level access permissions (Test cases 19-21).
-		// Any of the Bucket level access permissions shouldn't allow for DeleteObject operation  on an Anonymous/unsigned request.
-		{"s3:DeleteObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:DeleteObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:DeleteObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing DeleteObject access for different Object level access permissions (Test cases 22).
-		// read-only bucket policy is expected to not allow Delete Object operation on an anonymous request.
-		{"s3:DeleteObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write bucket policy is expected to allow Delete Bucket operation on an anonymous request (Test cases 23).
-		{"s3:DeleteObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  bucket policy is expected to allow Delete Object operation on an anonymous request (Test cases 24).
-		{"s3:DeleteObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
-
-		// s3:AbortMultipartUpload is the action necessary to be present in the bucket policy to allow
-		// cancelling or abortion of an already initiated multipart upload operation for an Anonymous/unsigned request.
-
-		// Cases for testing AbortMultipartUpload access for different Bucket level access permissions (Test cases 25-27).
-		// Any of the Bucket level access permissions shouldn't allow for AbortMultipartUpload operation on an Anonymous/unsigned request.
-		{"s3:AbortMultipartUpload", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:AbortMultipartUpload", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:AbortMultipartUpload", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing AbortMultipartUpload access for different Object level access permissions.
-		// read-only object policy is expected to not allow AbortMultipartUpload operation on an anonymous request (Test case 28).
-		{"s3:AbortMultipartUpload", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 29).
-		{"s3:AbortMultipartUpload", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 30).
-		{"s3:AbortMultipartUpload", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
-
-		// s3:PutObject is the action necessary to be present in the bucket policy to allow
-		// uploading of an object for an Anonymous/unsigned request.
-
-		// Cases for testing PutObject access for different Bucket level access permissions (Test cases 31-33).
-		// Any of the Bucket level access permissions shouldn't allow for PutObject operation on an Anonymous/unsigned request.
-		{"s3:PutObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:PutObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:PutObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing PutObject access for different Object level access permissions.
-		// read-only object policy is expected to not allow PutObject operation on an anonymous request (Test case 34).
-		{"s3:PutObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write object policy is expected to allow PutObject operation on an anonymous request (Test case 35).
-		{"s3:PutObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  object policy is expected to allow PutObject operation on an anonymous request (Test case 36).
-		{"s3:PutObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
-
-		// s3:GetObject is the action necessary to be present in the bucket policy to allow
-		// downloading of an object for an Anonymous/unsigned request.
-
-		// Cases for testing GetObject access for different Bucket level access permissions (Test cases 37-39).
-		// Any of the Bucket level access permissions shouldn't allow for GetObject operation on an Anonymous/unsigned request.
-		{"s3:GetObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:GetObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:GetObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing GetObject access for different Object level access permissions.
-		// read-only bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 40).
-		{"s3:GetObject", getReadOnlyObjectStatement(bucketName, objectPrefix), true},
-		// read-write bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 41).
-		{"s3:GetObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  bucket policy is expected to not allow  downloading of an Object on an anonymous request (Test case 42).
-		{"s3:GetObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:ListMultipartUploadParts is the action necessary to be present in the bucket policy to allow
-		// Listing of uploaded parts for an Anonymous/unsigned request.
-
-		// Any of the Bucket level access permissions shouldn't allow for ListMultipartUploadParts operation on an Anonymous/unsigned request.
-		// read-only bucket policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test cases 43-45).
-		{"s3:ListMultipartUploadParts", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:ListMultipartUploadParts", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:ListMultipartUploadParts", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// read-only object policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test case 46).
-		{"s3:ListMultipartUploadParts", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 47).
-		{"s3:ListMultipartUploadParts", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 48).
-		{"s3:ListMultipartUploadParts", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(
+				policy.GetBucketLocationAction,
+				policy.ListBucketMultipartUploadsAction,
+			),
+			policy.NewResourceSet(policy.NewResource(bucketName, "")),
+			condition.NewFunctions(),
+		)},
 	}
-	for i, testCase := range testCases {
-		actualResult := bucketPolicyActionMatch(testCase.action, testCase.statement)
-		if testCase.expectedResult != actualResult {
-			t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResult, actualResult)
-		}
+}
+
+func getAnonReadOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(policy.GetObjectAction),
+			policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+			condition.NewFunctions(),
+		)},
+	}
+}
+
+func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(
+				policy.AbortMultipartUploadAction,
+				policy.DeleteObjectAction,
+				policy.ListMultipartUploadPartsAction,
+				policy.PutObjectAction,
+			),
+			policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+			condition.NewFunctions(),
+		)},
 	}
 }
 
@@ -260,6 +110,8 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// template for constructing HTTP request body for PUT bucket policy.
 	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}`
 
+	bucketPolicyTemplateWithoutVersion := `{"Version":"","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}`
+
 	// test cases with sample input and expected output.
 	testCases := []struct {
 		bucketName string
@@ -290,7 +142,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			bucketName:         bucketName,
 			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
 
-			policyLen:          maxAccessPolicySize + 1,
+			policyLen:          maxBucketPolicySize + 1,
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			expectedRespStatus: http.StatusBadRequest,
@@ -357,7 +209,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		// Test case - 8.
 		// non-existent bucket is used.
 		// writing BucketPolicy should fail.
-		// should result is 500 InternalServerError.
+		// should result in 404 StatusNotFound
 		{
 			bucketName:         "non-existent-bucket",
 			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, "non-existent-bucket", "non-existent-bucket"))),
@@ -368,9 +220,9 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 9.
-		// invalid bucket name is used.
+		// non-existent bucket is used (with invalid bucket name)
 		// writing BucketPolicy should fail.
-		// should result is 400 StatusBadRequest.
+		// should result in 404 StatusNotFound
 		{
 			bucketName:         ".invalid-bucket",
 			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, ".invalid-bucket", ".invalid-bucket"))),
@@ -378,6 +230,19 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			policyLen:          len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
+			expectedRespStatus: http.StatusNotFound,
+		},
+		// Test case - 10.
+		// Existent bucket with policy with Version field empty.
+		// writing BucketPolicy should fail.
+		// should result in 400 StatusBadRequest.
+		{
+			bucketName:         bucketName,
+			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplateWithoutVersion, bucketName, bucketName))),
+
+			policyLen:          len(fmt.Sprintf(bucketPolicyTemplateWithoutVersion, bucketName, bucketName)),
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
 			expectedRespStatus: http.StatusBadRequest,
 		},
 	}
@@ -430,7 +295,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "PutBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "PutBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -459,7 +324,7 @@ func TestGetBucketPolicyHandler(t *testing.T) {
 func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
 	// template for constructing HTTP request body for PUT bucket policy.
-	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"],"Sid":""}]}`
+	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"]},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"]}]}`
 
 	// Writing bucket policy before running test on GetBucketPolicy.
 	putTestPolicies := []struct {
@@ -535,13 +400,13 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			expectedRespStatus:   http.StatusNotFound,
 		},
 		// Test case - 3.
-		// Case with invalid bucket name.
+		// Case with non-existent bucket name.
 		{
 			bucketName:           ".invalid-bucket-name",
 			accessKey:            credentials.AccessKey,
 			secretKey:            credentials.SecretKey,
 			expectedBucketPolicy: "",
-			expectedRespStatus:   http.StatusBadRequest,
+			expectedRespStatus:   http.StatusNotFound,
 		},
 	}
 	// Iterating over the cases, fetching the policy and validating the response.
@@ -572,7 +437,16 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 		if recV4.Code != testCase.expectedRespStatus {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			if expectedBucketPolicyStr != string(bucketPolicyReadBuf) {
+			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+
+			if !reflect.DeepEqual(expectedPolicy, gotPolicy) {
 				t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType)
 			}
 		}
@@ -598,7 +472,16 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		}
 		if recV2.Code == http.StatusOK {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			if expectedBucketPolicyStr != string(bucketPolicyReadBuf) {
+			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+
+			if !reflect.DeepEqual(expectedPolicy, gotPolicy) {
 				t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType)
 			}
 		}
@@ -617,7 +500,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "GetBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "GetBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -742,12 +625,12 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 3.
-		// Case with invalid bucket name.
+		// Case with non-existent-bucket.
 		{
 			bucketName:         ".invalid-bucket-name",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusBadRequest,
+			expectedRespStatus: http.StatusNotFound,
 		},
 	}
 	// Iterating over the cases and deleting the bucket policy and then asserting response.
@@ -820,7 +703,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "DeleteBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "DeleteBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -838,174 +721,3 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	// `ExecObjectLayerAPINilTest` manages the operation.
 	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
 }
-
-// TestBucketPolicyConditionMatch - Tests to validate whether bucket policy conditions match.
-func TestBucketPolicyConditionMatch(t *testing.T) {
-	// obtain the inner map[string]set.StringSet for policy.Statement.Conditions.
-	getInnerMap := func(key2, value string) map[string]set.StringSet {
-		innerMap := make(map[string]set.StringSet)
-		innerMap[key2] = set.CreateStringSet(value)
-		return innerMap
-	}
-
-	// obtain policy.Statement with Conditions set.
-	getStatementWithCondition := func(key1, key2, value string) policy.Statement {
-		innerMap := getInnerMap(key2, value)
-		// to set policyStatment.Conditions .
-		conditions := make(policy.ConditionMap)
-		conditions[key1] = innerMap
-		// new policy statement.
-		statement := policy.Statement{}
-		// set the condition.
-		statement.Conditions = conditions
-		return statement
-	}
-
-	testCases := []struct {
-		statementCondition policy.Statement
-		condition          map[string]set.StringSet
-
-		expectedMatch bool
-	}{
-
-		// Test case - 1.
-		// StringEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Asia/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 2.
-		// StringEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Africa/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 3.
-		// StringEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Asia/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 4.
-		// StringEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Africa/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 5.
-		// StringNotEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Asia/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 6.
-		// StringNotEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Africa/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 7.
-		// StringNotEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Asia/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 8.
-		// StringNotEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Africa/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 9.
-		// StringLike condition matches.
-		{
-			statementCondition: getStatementWithCondition("StringLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "http://www.example.com/"),
-			expectedMatch:      true,
-		},
-		// Test case - 10.
-		// StringLike condition doesn't match.
-		{
-			statementCondition: getStatementWithCondition("StringLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "www.somethingelse.com"),
-			expectedMatch:      false,
-		},
-		// Test case - 11.
-		// StringNotLike condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("StringNotLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "http://www.example.com/"),
-			expectedMatch:      false,
-		},
-		// Test case - 12.
-		// StringNotLike condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("StringNotLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "http://somethingelse.com/"),
-			expectedMatch:      true,
-		},
-		// Test case 13.
-		// IpAddress condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.143.2"),
-			expectedMatch:      true,
-		},
-		// Test case 14.
-		// IpAddress condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "127.240.143.224"),
-			expectedMatch:      false,
-		},
-		// Test case 15.
-		// NotIpAddress condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.144.188"),
-			expectedMatch:      true,
-		},
-		// Test case 16.
-		// NotIpAddress condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.143.243"),
-			expectedMatch:      false,
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("Case %d", i+1), func(t *testing.T) {
-			// call the function under test and assert the result with the expected result.
-			doesMatch := bucketPolicyConditionMatch(tc.condition, tc.statementCondition)
-			if tc.expectedMatch != doesMatch {
-				t.Errorf("Expected the match to be `%v`; got `%v` - %v %v.",
-					tc.expectedMatch, doesMatch, tc.condition, tc.statementCondition)
-			}
-		})
-	}
-}

cmd/bucket-policy-parser.go

@@ -1,295 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Package cmd This file implements AWS Access Policy Language parser in
-// accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
-package cmd
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"sort"
-	"strings"
-
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
-)
-
-var emptyBucketPolicy = policy.BucketAccessPolicy{}
-
-var conditionKeyActionMap = policy.ConditionKeyMap{
-	"s3:prefix": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads"),
-	"s3:max-keys": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads",
-		"s3:ListMultipartUploadParts"),
-}
-
-// supportedActionMap - lists all the actions supported by minio.
-var supportedActionMap = set.CreateStringSet("*", "s3:*", "s3:GetObject",
-	"s3:ListBucket", "s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
-	"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts")
-
-// supported Conditions type.
-var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals", "StringLike", "StringNotLike", "IpAddress", "NotIpAddress")
-
-// Validate s3:prefix, s3:max-keys are present if not
-// supported keys for the conditions.
-var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aws:Referer", "aws:SourceIp")
-
-// supportedEffectMap - supported effects.
-var supportedEffectMap = set.CreateStringSet("Allow", "Deny")
-
-// isValidActions - are actions valid.
-func isValidActions(actions set.StringSet) (err error) {
-	// Statement actions cannot be empty.
-	if actions.IsEmpty() {
-		err = errors.New("Action list cannot be empty")
-		return err
-	}
-	if unsupportedActions := actions.Difference(supportedActionMap); !unsupportedActions.IsEmpty() {
-		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document",
-			unsupportedActions)
-		return err
-	}
-	return nil
-}
-
-// isValidEffect - is effect valid.
-func isValidEffect(effect string) (err error) {
-	// Statement effect cannot be empty.
-	if effect == "" {
-		err = errors.New("Policy effect cannot be empty")
-		return err
-	}
-	if !supportedEffectMap.Contains(effect) {
-		err = errors.New("Unsupported Effect found: ‘" + effect + "’, please validate your policy document")
-		return err
-	}
-	return nil
-}
-
-// isValidResources - are valid resources.
-func isValidResources(resources set.StringSet) (err error) {
-	// Statement resources cannot be empty.
-	if resources.IsEmpty() {
-		err = errors.New("Resource list cannot be empty")
-		return err
-	}
-	for resource := range resources {
-		if !hasPrefix(resource, bucketARNPrefix) {
-			err = errors.New("Unsupported resource style found: ‘" + resource + "’, please validate your policy document")
-			return err
-		}
-		resourceSuffix := strings.SplitAfter(resource, bucketARNPrefix)[1]
-		if len(resourceSuffix) == 0 || hasPrefix(resourceSuffix, "/") {
-			err = errors.New("Invalid resource style found: ‘" + resource + "’, please validate your policy document")
-			return err
-		}
-	}
-	return nil
-}
-
-// isValidPrincipals - are valid principals.
-func isValidPrincipals(principal policy.User) (err error) {
-	if principal.AWS.IsEmpty() {
-		return errors.New("Principal cannot be empty")
-	}
-	if diff := principal.AWS.Difference(set.CreateStringSet("*")); !diff.IsEmpty() {
-		// Minio does not support or implement IAM, "*" is the only valid value.
-		// Amazon s3 doc on principal:
-		// http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
-		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document",
-			diff)
-		return err
-	}
-	return nil
-}
-
-// isValidConditions - returns nil if the given conditions valid and
-// corresponding error otherwise.
-func isValidConditions(actions set.StringSet, conditions policy.ConditionMap) (err error) {
-	// Verify conditions should be valid. Validate if only
-	// supported condition keys are present and return error
-	// otherwise.
-	conditionKeyVal := make(map[string]set.StringSet)
-	for conditionType := range conditions {
-		if !supportedConditionsType.Contains(conditionType) {
-			err = fmt.Errorf("Unsupported condition type '%s', please validate your policy document", conditionType)
-			return err
-		}
-		for key, value := range conditions[conditionType] {
-			if !supportedConditionsKey.Contains(key) {
-				err = fmt.Errorf("Unsupported condition key '%s', please validate your policy document", conditionType)
-				return err
-			}
-
-			compatibleActions := conditionKeyActionMap[key]
-			if !compatibleActions.IsEmpty() &&
-				compatibleActions.Intersection(actions).IsEmpty() {
-				err = fmt.Errorf("Unsupported condition key %s for the given actions %s, "+
-					"please validate your policy document", key, actions)
-				return err
-			}
-
-			conditionVal, ok := conditionKeyVal[key]
-			if ok && !value.Intersection(conditionVal).IsEmpty() {
-				err = fmt.Errorf("Ambigious condition values for key '%s', please validate your policy document", key)
-				return err
-			}
-			conditionKeyVal[key] = value
-		}
-	}
-	return nil
-}
-
-// List of actions for which prefixes are not allowed.
-var invalidPrefixActions = set.StringSet{
-	"s3:GetBucketLocation":          {},
-	"s3:ListBucket":                 {},
-	"s3:ListBucketMultipartUploads": {},
-	// Add actions which do not honor prefixes.
-}
-
-// resourcePrefix - provides the prefix removing any wildcards.
-func resourcePrefix(resource string) string {
-	if strings.HasSuffix(resource, "*") {
-		resource = strings.TrimSuffix(resource, "*")
-	}
-	return resource
-}
-
-// checkBucketPolicyResources validates Resources in unmarshalled bucket policy structure.
-// - Resources are validated against the given set of Actions.
-// -
-func checkBucketPolicyResources(bucket string, bucketPolicy policy.BucketAccessPolicy) APIErrorCode {
-	// Validate statements for special actions and collect resources
-	// for others to validate nesting.
-	var resourceMap = set.NewStringSet()
-	for _, statement := range bucketPolicy.Statements {
-		for action := range statement.Actions {
-			for resource := range statement.Resources {
-				resourcePrefix := strings.SplitAfter(resource, bucketARNPrefix)[1]
-				if _, ok := invalidPrefixActions[action]; ok {
-					// Resource prefix is not equal to bucket for
-					// prefix invalid actions, reject them.
-					if resourcePrefix != bucket {
-						return ErrMalformedPolicy
-					}
-				} else {
-					// For all other actions validate if resourcePrefix begins
-					// with bucket name, if not reject them.
-					if strings.Split(resourcePrefix, "/")[0] != bucket {
-						return ErrMalformedPolicy
-					}
-					// All valid resources collect them separately to verify nesting.
-					resourceMap.Add(resourcePrefix)
-				}
-			}
-		}
-	}
-
-	var resources []string
-	for resource := range resourceMap {
-		resources = append(resources, resourcePrefix(resource))
-	}
-
-	// Sort strings as shorter first.
-	sort.Strings(resources)
-
-	for len(resources) > 1 {
-		var resource string
-		resource, resources = resources[0], resources[1:]
-		// Loop through all resources, if one of them matches with
-		// previous shorter one, it means we have detected
-		// nesting. Reject such rules.
-		for _, otherResource := range resources {
-			// Common prefix reject such rules.
-			if hasPrefix(otherResource, resource) {
-				return ErrPolicyNesting
-			}
-		}
-	}
-
-	// No errors found.
-	return ErrNone
-}
-
-// parseBucketPolicy - parses and validates if bucket policy is of
-// proper JSON and follows allowed restrictions with policy standards.
-func parseBucketPolicy(bucketPolicyReader io.Reader, bktPolicy *policy.BucketAccessPolicy) (err error) {
-	// Parse bucket policy reader.
-	decoder := json.NewDecoder(bucketPolicyReader)
-	if err = decoder.Decode(bktPolicy); err != nil {
-		return err
-	}
-
-	// Policy version cannot be empty.
-	if len(bktPolicy.Version) == 0 {
-		err = errors.New("Policy version cannot be empty")
-		return err
-	}
-
-	// Policy statements cannot be empty.
-	if len(bktPolicy.Statements) == 0 {
-		err = errors.New("Policy statement cannot be empty")
-		return err
-	}
-
-	// Loop through all policy statements and validate entries.
-	for _, statement := range bktPolicy.Statements {
-		// Statement effect should be valid.
-		if err := isValidEffect(statement.Effect); err != nil {
-			return err
-		}
-		// Statement principal should be supported format.
-		if err := isValidPrincipals(statement.Principal); err != nil {
-			return err
-		}
-		// Statement actions should be valid.
-		if err := isValidActions(statement.Actions); err != nil {
-			return err
-		}
-		// Statement resources should be valid.
-		if err := isValidResources(statement.Resources); err != nil {
-			return err
-		}
-		// Statement conditions should be valid.
-		if err := isValidConditions(statement.Actions, statement.Conditions); err != nil {
-			return err
-		}
-	}
-
-	// Separate deny and allow statements, so that we can apply deny
-	// statements in the beginning followed by Allow statements.
-	var denyStatements []policy.Statement
-	var allowStatements []policy.Statement
-	for _, statement := range bktPolicy.Statements {
-		if statement.Effect == "Deny" {
-			denyStatements = append(denyStatements, statement)
-			continue
-		}
-
-		// else if statement.Effect == "Allow"
-		allowStatements = append(allowStatements, statement)
-	}
-
-	// Deny statements are enforced first once matched.
-	bktPolicy.Statements = append(denyStatements, allowStatements...)
-
-	// Return successfully parsed policy structure.
-	return nil
-}

cmd/bucket-policy-parser_test.go

@@ -1,867 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package cmd
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"reflect"
-	"testing"
-
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
-)
-
-// Common bucket actions for both read and write policies.
-var (
-	readWriteBucketActions = []string{
-		"s3:GetBucketLocation",
-		"s3:ListBucket",
-		"s3:ListBucketMultipartUploads",
-		// Add more bucket level read-write actions here.
-	}
-	readWriteObjectActions = []string{
-		"s3:AbortMultipartUpload",
-		"s3:DeleteObject",
-		"s3:GetObject",
-		"s3:ListMultipartUploadParts",
-		"s3:PutObject",
-		// Add more object level read-write actions here.
-	}
-)
-
-// Write only actions.
-var (
-	writeOnlyBucketActions = []string{
-		"s3:GetBucketLocation",
-		"s3:ListBucketMultipartUploads",
-		// Add more bucket level write actions here.
-	}
-	writeOnlyObjectActions = []string{
-		"s3:AbortMultipartUpload",
-		"s3:DeleteObject",
-		"s3:ListMultipartUploadParts",
-		"s3:PutObject",
-		// Add more object level write actions here.
-	}
-)
-
-// Read only actions.
-var (
-	readOnlyBucketActions = []string{
-		"s3:GetBucketLocation",
-		"s3:ListBucket",
-		// Add more bucket level read actions here.
-	}
-	readOnlyObjectActions = []string{
-		"s3:GetObject",
-		// Add more object level read actions here.
-	}
-)
-
-// Obtain bucket statement for read-write bucketPolicy.
-func getReadWriteObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
-	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
-	objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...)
-	return objectResourceStatement
-}
-
-// Obtain object statement for read-write bucketPolicy.
-func getReadWriteBucketStatement(bucketName, objectPrefix string) policy.Statement {
-	bucketResourceStatement := policy.Statement{}
-	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
-	bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...)
-	return bucketResourceStatement
-}
-
-// Obtain statements for read-write bucketPolicy.
-func getReadWriteStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
-	// Save the read write policy.
-	statements = append(statements, getReadWriteBucketStatement(bucketName, objectPrefix), getReadWriteObjectStatement(bucketName, objectPrefix))
-	return statements
-}
-
-// Obtain bucket statement for read only bucketPolicy.
-func getReadOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
-	bucketResourceStatement := policy.Statement{}
-	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
-	bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...)
-	return bucketResourceStatement
-}
-
-// Obtain object statement for read only bucketPolicy.
-func getReadOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
-	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
-	objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...)
-	return objectResourceStatement
-}
-
-// Obtain statements for read only bucketPolicy.
-func getReadOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
-	// Save the read only policy.
-	statements = append(statements, getReadOnlyBucketStatement(bucketName, objectPrefix), getReadOnlyObjectStatement(bucketName, objectPrefix))
-	return statements
-}
-
-// Obtain bucket statements for write only bucketPolicy.
-func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
-
-	bucketResourceStatement := policy.Statement{}
-	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
-	bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...)
-	return bucketResourceStatement
-}
-
-// Obtain object statements for write only bucketPolicy.
-func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
-	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
-	objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...)
-	return objectResourceStatement
-}
-
-// Obtain statements for write only bucketPolicy.
-func getWriteOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
-	// Write only policy.
-	// Save the write only policy.
-	statements = append(statements, getWriteOnlyBucketStatement(bucketName, objectPrefix), getWriteOnlyBucketStatement(bucketName, objectPrefix))
-	return statements
-}
-
-// Tests validate Action validator.
-func TestIsValidActions(t *testing.T) {
-	testCases := []struct {
-		// input.
-		actions set.StringSet
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Action.
-		// Test case - 1.
-		// "s3:ListObject" is an invalid Action.
-		{set.CreateStringSet([]string{"s3:GetObject", "s3:ListObject", "s3:RemoveObject"}...),
-			errors.New("Unsupported actions found: ‘set.StringSet{\"s3:RemoveObject\":struct {}{}, \"s3:ListObject\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 2.
-		// Empty Actions.
-		{set.CreateStringSet([]string{}...), errors.New("Action list cannot be empty"), false},
-		// Test case - 3.
-		// "s3:DeleteEverything"" is an invalid Action.
-		{set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...),
-			errors.New("Unsupported actions found: ‘set.StringSet{\"s3:DeleteEverything\":struct {}{}}’, please validate your policy document"), false},
-		// Inputs with valid Action.
-		// Test Case - 4.
-		{set.CreateStringSet([]string{
-			"s3:*", "*", "s3:GetObject", "s3:ListBucket",
-			"s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
-			"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads",
-			"s3:ListMultipartUploadParts"}...), nil, true},
-	}
-	for i, testCase := range testCases {
-		err := isValidActions(testCase.actions)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-	}
-}
-
-// Tests validate Effect validator.
-func TestIsValidEffect(t *testing.T) {
-	testCases := []struct {
-		// input.
-		effect string
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Effect.
-		// Test case - 1.
-		{"", errors.New("Policy effect cannot be empty"), false},
-		// Test case - 2.
-		{"DontAllow", errors.New("Unsupported Effect found: ‘DontAllow’, please validate your policy document"), false},
-		// Test case - 3.
-		{"NeverAllow", errors.New("Unsupported Effect found: ‘NeverAllow’, please validate your policy document"), false},
-		// Test case - 4.
-		{"AllowAlways", errors.New("Unsupported Effect found: ‘AllowAlways’, please validate your policy document"), false},
-
-		// Inputs with valid Effect.
-		// Test Case - 5.
-		{"Allow", nil, true},
-		// Test Case - 6.
-		{"Deny", nil, true},
-	}
-	for i, testCase := range testCases {
-		err := isValidEffect(testCase.effect)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-	}
-}
-
-// Tests validate Resources validator.
-func TestIsValidResources(t *testing.T) {
-	testCases := []struct {
-		// input.
-		resources []string
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Action.
-		// Test case - 1.
-		// Empty Resources.
-		{[]string{}, errors.New("Resource list cannot be empty"), false},
-		// Test case - 2.
-		// A valid resource should have prefix bucketARNPrefix.
-		{[]string{"my-resource"}, errors.New("Unsupported resource style found: ‘my-resource’, please validate your policy document"), false},
-		// Test case - 3.
-		// A Valid resource should have bucket name followed by bucketARNPrefix.
-		{[]string{bucketARNPrefix}, errors.New("Invalid resource style found: ‘arn:aws:s3:::’, please validate your policy document"), false},
-		// Test Case - 4.
-		// Valid resource shouldn't have slash('/') followed by bucketARNPrefix.
-		{[]string{bucketARNPrefix + "/"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::/’, please validate your policy document"), false},
-
-		// Test cases with valid Resources.
-		{[]string{bucketARNPrefix + "my-bucket"}, nil, true},
-		{[]string{bucketARNPrefix + "my-bucket/Asia/*"}, nil, true},
-		{[]string{bucketARNPrefix + "my-bucket/Asia/India/*"}, nil, true},
-	}
-	for i, testCase := range testCases {
-		err := isValidResources(set.CreateStringSet(testCase.resources...))
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-	}
-}
-
-// Tests validate principals validator.
-func TestIsValidPrincipals(t *testing.T) {
-	testCases := []struct {
-		// input.
-		principals []string
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Principals.
-		// Test case - 1.
-		// Empty Principals list.
-		{[]string{}, errors.New("Principal cannot be empty"), false},
-		// Test case - 2.
-		// "*" is the only valid principal.
-		{[]string{"my-principal"}, errors.New("Unsupported principals found: ‘set.StringSet{\"my-principal\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 3.
-		{[]string{"*", "111122233"}, errors.New("Unsupported principals found: ‘set.StringSet{\"111122233\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 4.
-		// Test case with valid principal value.
-		{[]string{"*"}, nil, true},
-	}
-	for i, testCase := range testCases {
-		u := policy.User{
-			AWS: set.CreateStringSet(testCase.principals...),
-		}
-		err := isValidPrincipals(u)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-	}
-}
-
-// getEmptyConditionMap - returns a function that generates a
-// condition key map for a given key.
-func getEmptyConditionMap(conditionKey string) func() policy.ConditionMap {
-	emptyConditonGenerator := func() policy.ConditionMap {
-		emptyMap := make(policy.ConditionKeyMap)
-		conditions := make(policy.ConditionMap)
-		conditions[conditionKey] = emptyMap
-		return conditions
-	}
-	return emptyConditonGenerator
-}
-
-// Tests validate policy.Statement condition validator.
-func TestIsValidConditions(t *testing.T) {
-	// returns empty conditions map.
-	setEmptyConditions := func() policy.ConditionMap {
-		return make(policy.ConditionMap)
-	}
-
-	// returns map with the "StringEquals" set to empty map.
-	setEmptyStringEquals := getEmptyConditionMap("StringEquals")
-
-	// returns map with the "StringNotEquals" set to empty map.
-	setEmptyStringNotEquals := getEmptyConditionMap("StringNotEquals")
-
-	// returns map with the "StringLike" set to empty map.
-	setEmptyStringLike := getEmptyConditionMap("StringLike")
-
-	// returns map with the "StringNotLike" set to empty map.
-	setEmptyStringNotLike := getEmptyConditionMap("StringNotLike")
-
-	// returns map with the "IpAddress" set to empty map.
-	setEmptyIPAddress := getEmptyConditionMap("IpAddress")
-
-	// returns map with "NotIpAddress" set to empty map.
-	setEmptyNotIPAddress := getEmptyConditionMap("NotIpAddress")
-
-	// Generate conditions.
-	generateConditions := func(key1, key2, value string) policy.ConditionMap {
-		innerMap := make(policy.ConditionKeyMap)
-		innerMap[key2] = set.CreateStringSet(value)
-		conditions := make(policy.ConditionMap)
-		conditions[key1] = innerMap
-		return conditions
-	}
-
-	// generate ambigious conditions.
-	generateAmbigiousConditions := func() policy.ConditionMap {
-		prefixMap := make(policy.ConditionKeyMap)
-		prefixMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
-		conditions["StringEquals"] = prefixMap
-		conditions["StringNotEquals"] = prefixMap
-		return conditions
-	}
-
-	// generate valid and non valid type in the condition map.
-	generateValidInvalidConditions := func() policy.ConditionMap {
-		innerMap := make(policy.ConditionKeyMap)
-		innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
-		conditions["StringEquals"] = innerMap
-		conditions["InvalidType"] = innerMap
-		return conditions
-	}
-
-	// generate valid and invalid keys for valid types in the same condition map.
-	generateValidInvalidConditionKeys := func() policy.ConditionMap {
-		innerMapValid := make(policy.ConditionKeyMap)
-		innerMapValid["s3:prefix"] = set.CreateStringSet("Asia/")
-		innerMapInValid := make(map[string]set.StringSet)
-		innerMapInValid["s3:invalid"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
-		conditions["StringEquals"] = innerMapValid
-		conditions["StringEquals"] = innerMapInValid
-		return conditions
-	}
-
-	// List of Conditions used for test cases.
-	testConditions := []policy.ConditionMap{
-		generateConditions("StringValues", "s3:max-keys", "100"),
-		generateConditions("StringEquals", "s3:Object", "100"),
-		generateAmbigiousConditions(),
-		generateValidInvalidConditions(),
-		generateValidInvalidConditionKeys(),
-		setEmptyConditions(),
-		setEmptyStringEquals(),
-		setEmptyStringNotEquals(),
-		setEmptyStringLike(),
-		setEmptyStringNotLike(),
-		setEmptyIPAddress(),
-		setEmptyNotIPAddress(),
-		generateConditions("StringEquals", "s3:prefix", "Asia/"),
-		generateConditions("StringEquals", "s3:max-keys", "100"),
-		generateConditions("StringNotEquals", "s3:prefix", "Asia/"),
-		generateConditions("StringNotEquals", "s3:max-keys", "100"),
-	}
-
-	getObjectActionSet := set.CreateStringSet("s3:GetObject")
-	roBucketActionSet := set.CreateStringSet(readOnlyBucketActions...)
-	maxKeysConditionErr := fmt.Errorf("Unsupported condition key %s for the given actions %s, "+
-		"please validate your policy document", "s3:max-keys", getObjectActionSet)
-	testCases := []struct {
-		inputActions   set.StringSet
-		inputCondition policy.ConditionMap
-		// expected result.
-		expectedErr error
-		// flag indicating whether test should pass.
-		shouldPass bool
-	}{
-		// Malformed conditions.
-		// Test case - 1.
-		// "StringValues" is an invalid type.
-		{roBucketActionSet, testConditions[0], fmt.Errorf("Unsupported condition type 'StringValues', " +
-			"please validate your policy document"), false},
-		// Test case - 2.
-		// "s3:Object" is an invalid key.
-		{roBucketActionSet, testConditions[1], fmt.Errorf("Unsupported condition key " +
-			"'StringEquals', please validate your policy document"), false},
-		// Test case - 3.
-		// Test case with Ambigious conditions set.
-		{roBucketActionSet, testConditions[2], fmt.Errorf("Ambigious condition values for key 's3:prefix', " +
-			"please validate your policy document"), false},
-		// Test case - 4.
-		// Test case with valid and invalid condition types.
-		{roBucketActionSet, testConditions[3], fmt.Errorf("Unsupported condition type 'InvalidType', " +
-			"please validate your policy document"), false},
-		// Test case - 5.
-		// Test case with valid and invalid condition keys.
-		{roBucketActionSet, testConditions[4], fmt.Errorf("Unsupported condition key 'StringEquals', " +
-			"please validate your policy document"), false},
-		// Test cases with valid conditions.
-		// Test case - 6.
-		{roBucketActionSet, testConditions[5], nil, true},
-		// Test case - 7.
-		{roBucketActionSet, testConditions[6], nil, true},
-		// Test case - 8.
-		{roBucketActionSet, testConditions[7], nil, true},
-		// Test case - 9.
-		{roBucketActionSet, testConditions[8], nil, true},
-		// Test case - 10.
-		{roBucketActionSet, testConditions[9], nil, true},
-		// Test case - 11.
-		{roBucketActionSet, testConditions[10], nil, true},
-		// Test case - 12.
-		{roBucketActionSet, testConditions[11], nil, true},
-		// Test case - 13.
-		{roBucketActionSet, testConditions[12], nil, true},
-		// Test case - 11.
-		{roBucketActionSet, testConditions[13], nil, true},
-		// Test case - 12.
-		{roBucketActionSet, testConditions[14], nil, true},
-		// Test case - 13.
-		{getObjectActionSet, testConditions[15], maxKeysConditionErr, false},
-	}
-	for i, testCase := range testCases {
-		actualErr := isValidConditions(testCase.inputActions, testCase.inputCondition)
-		if actualErr != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, actualErr.Error())
-		}
-		if actualErr == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.expectedErr.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if actualErr != nil && !testCase.shouldPass {
-			if actualErr.Error() != testCase.expectedErr.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.expectedErr.Error(), actualErr.Error())
-			}
-		}
-	}
-}
-
-// Tests validate Policy Action and Resource fields.
-func TestCheckbucketPolicyResources(t *testing.T) {
-	// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
-	setValidPrefixActions := func(statements []policy.Statement) []policy.Statement {
-		statements[0].Actions = set.CreateStringSet([]string{"s3:DeleteObject", "s3:PutObject"}...)
-		return statements
-	}
-	// contracting policy statement with recursive resources.
-	// should result in ErrMalformedPolicy
-	setRecurseResource := func(statements []policy.Statement) []policy.Statement {
-		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}...)
-		return statements
-	}
-
-	// constructing policy statement with lexically close characters.
-	// should not result in ErrMalformedPolicy
-	setResourceLexical := func(statements []policy.Statement) []policy.Statement {
-		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}...)
-		return statements
-	}
-
-	// List of bucketPolicy used for tests.
-	bucketAccessPolicies := []policy.BucketAccessPolicy{
-		// bucketPolicy - 1.
-		// Contains valid read only policy statement.
-		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
-		// bucketPolicy - 2.
-		// Contains valid read-write only policy statement.
-		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
-		// bucketPolicy - 3.
-		// Contains valid write only policy statement.
-		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
-		// bucketPolicy - 4.
-		// Contains invalidPrefixActions.
-		// Since resourcePrefix is not to the bucket-name, it return ErrMalformedPolicy.
-		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket-fail", "Asia/India/")},
-		// bucketPolicy - 5.
-		// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
-		// but bucket part of the resource is not equal to the bucket name.
-		// this results in return of ErrMalformedPolicy.
-		{Version: "1.0", Statements: setValidPrefixActions(getWriteOnlyStatement("minio-bucket-fail", "Asia/India/"))},
-		// bucketPolicy - 6.
-		// contracting policy statement with recursive resources.
-		// should result in ErrMalformedPolicy
-		{Version: "1.0", Statements: setRecurseResource(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "")))},
-		// BucketPolciy - 7.
-		// constructing policy statement with non recursive but
-		// lexically close resources.
-		// should result in ErrNone.
-		{Version: "1.0", Statements: setResourceLexical(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "oo")))},
-	}
-
-	testCases := []struct {
-		inputPolicy policy.BucketAccessPolicy
-		// expected results.
-		apiErrCode APIErrorCode
-		// Flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Test case - 1.
-		{bucketAccessPolicies[0], ErrNone, true},
-		// Test case - 2.
-		{bucketAccessPolicies[1], ErrNone, true},
-		// Test case - 3.
-		{bucketAccessPolicies[2], ErrNone, true},
-		// Test case - 4.
-		// contains invalidPrefixActions (check bucket-policy-parser.go).
-		// Resource prefix will not be equal to the bucket name in this case.
-		{bucketAccessPolicies[3], ErrMalformedPolicy, false},
-		// Test case - 5.
-		// actions contain invalidPrefixActions (check bucket-policy-parser.go).
-		// Resource prefix bucket part is not equal to the bucket name in this case.
-		{bucketAccessPolicies[4], ErrMalformedPolicy, false},
-		// Test case - 6.
-		// contracting policy statement with recursive resources.
-		// should result in ErrPolicyNesting.
-		{bucketAccessPolicies[5], ErrPolicyNesting, false},
-		// Test case - 7.
-		// constructing policy statement with lexically close
-		// characters.
-		// should result in ErrNone.
-		{bucketAccessPolicies[6], ErrNone, true},
-	}
-	for i, testCase := range testCases {
-		apiErrCode := checkBucketPolicyResources("minio-bucket", testCase.inputPolicy)
-		if apiErrCode != ErrNone && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with Errocode %v", i+1, apiErrCode)
-		}
-		if apiErrCode == ErrNone && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with ErrCode %v, but passed instead", i+1, testCase.apiErrCode)
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if apiErrCode != ErrNone && !testCase.shouldPass {
-			if testCase.apiErrCode != apiErrCode {
-				t.Errorf("Test %d: Expected to fail with error code %v, but instead failed with error code %v", i+1, testCase.apiErrCode, apiErrCode)
-			}
-		}
-	}
-}
-
-// Tests validate parsing of BucketAccessPolicy.
-func TestParseBucketPolicy(t *testing.T) {
-	// set Unsupported Actions.
-	setUnsupportedActions := func(statements []policy.Statement) []policy.Statement {
-		// "s3:DeleteEverything"" is an Unsupported Action.
-		statements[0].Actions = set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...)
-		return statements
-	}
-	// set unsupported Effect.
-	setUnsupportedEffect := func(statements []policy.Statement) []policy.Statement {
-		// Effect "Don't allow" is Unsupported.
-		statements[0].Effect = "DontAllow"
-		return statements
-	}
-	// set unsupported principals.
-	setUnsupportedPrincipals := func(statements []policy.Statement) []policy.Statement {
-		// "User1111"" is an Unsupported Principal.
-		statements[0].Principal = policy.User{
-			AWS: set.CreateStringSet([]string{"*", "User1111"}...),
-		}
-		return statements
-	}
-	// set unsupported Resources.
-	setUnsupportedResources := func(statements []policy.Statement) []policy.Statement {
-		// "s3:DeleteEverything"" is an Unsupported Action.
-		statements[0].Resources = set.CreateStringSet([]string{"my-resource"}...)
-		return statements
-	}
-	// List of bucketPolicy used for test cases.
-	bucketAccesPolicies := []policy.BucketAccessPolicy{
-		// bucketPolicy - 0.
-		// bucketPolicy statement empty.
-		{Version: "1.0"},
-		// bucketPolicy - 1.
-		// bucketPolicy version empty.
-		{Version: "", Statements: []policy.Statement{}},
-		// bucketPolicy - 2.
-		// Readonly bucketPolicy.
-		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
-		// bucketPolicy - 3.
-		// Read-Write bucket policy.
-		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
-		// bucketPolicy - 4.
-		// Write only bucket policy.
-		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
-		// bucketPolicy - 5.
-		// bucketPolicy statement contains unsupported action.
-		{Version: "1.0", Statements: setUnsupportedActions(getReadOnlyStatement("minio-bucket", ""))},
-		// bucketPolicy - 6.
-		// bucketPolicy statement contains unsupported Effect.
-		{Version: "1.0", Statements: setUnsupportedEffect(getReadWriteStatement("minio-bucket", "Asia/"))},
-		// bucketPolicy - 7.
-		// bucketPolicy statement contains unsupported Principal.
-		{Version: "1.0", Statements: setUnsupportedPrincipals(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
-		// bucketPolicy - 8.
-		// bucketPolicy statement contains unsupported Resource.
-		{Version: "1.0", Statements: setUnsupportedResources(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
-	}
-
-	testCases := []struct {
-		inputPolicy policy.BucketAccessPolicy
-		// expected results.
-		expectedPolicy policy.BucketAccessPolicy
-		err            error
-		// Flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Test case - 1.
-		// bucketPolicy statement empty.
-		{bucketAccesPolicies[0], policy.BucketAccessPolicy{}, errors.New("Policy statement cannot be empty"), false},
-		// Test case - 2.
-		// bucketPolicy version empty.
-		{bucketAccesPolicies[1], policy.BucketAccessPolicy{}, errors.New("Policy version cannot be empty"), false},
-		// Test case - 3.
-		// Readonly bucketPolicy.
-		{bucketAccesPolicies[2], bucketAccesPolicies[2], nil, true},
-		// Test case - 4.
-		// Read-Write bucket policy.
-		{bucketAccesPolicies[3], bucketAccesPolicies[3], nil, true},
-		// Test case - 5.
-		// Write only bucket policy.
-		{bucketAccesPolicies[4], bucketAccesPolicies[4], nil, true},
-		// Test case - 6.
-		// bucketPolicy statement contains unsupported action.
-		{bucketAccesPolicies[5], bucketAccesPolicies[5], fmt.Errorf("Unsupported actions found: ‘set.StringSet{\"s3:DeleteEverything\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 7.
-		// bucketPolicy statement contains unsupported Effect.
-		{bucketAccesPolicies[6], bucketAccesPolicies[6], fmt.Errorf("Unsupported Effect found: ‘DontAllow’, please validate your policy document"), false},
-		// Test case - 8.
-		// bucketPolicy statement contains unsupported Principal.
-		{bucketAccesPolicies[7], bucketAccesPolicies[7], fmt.Errorf("Unsupported principals found: ‘set.StringSet{\"User1111\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 9.
-		// bucketPolicy statement contains unsupported Resource.
-		{bucketAccesPolicies[8], bucketAccesPolicies[8], fmt.Errorf("Unsupported resource style found: ‘my-resource’, please validate your policy document"), false},
-	}
-	for i, testCase := range testCases {
-		var buffer bytes.Buffer
-		encoder := json.NewEncoder(&buffer)
-		err := encoder.Encode(testCase.inputPolicy)
-		if err != nil {
-			t.Fatalf("Test %d: Couldn't Marshal bucket policy %s", i+1, err)
-		}
-
-		var actualAccessPolicy = policy.BucketAccessPolicy{}
-		err = parseBucketPolicy(&buffer, &actualAccessPolicy)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-		// Test passes as expected, but the output values are verified for correctness here.
-		if err == nil && testCase.shouldPass {
-			if !reflect.DeepEqual(testCase.expectedPolicy, actualAccessPolicy) {
-				t.Errorf("Test %d: The expected statements from resource statement generator doesn't match the actual statements", i+1)
-			}
-		}
-	}
-}
-
-func TestAWSRefererCondition(t *testing.T) {
-	resource := set.CreateStringSet([]string{
-		fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/"+"Asia"+"*"),
-	}...)
-
-	conditionsKeyMap := make(policy.ConditionKeyMap)
-	conditionsKeyMap.Add("aws:Referer",
-		set.CreateStringSet("www.example.com",
-			"http://www.example.com"))
-
-	requestConditionMap := make(policy.ConditionKeyMap)
-	requestConditionMap["referer"] = set.CreateStringSet("www.example.com")
-
-	testCases := []struct {
-		effect       string
-		conditionKey string
-		match        bool
-	}{
-		{
-			effect:       "Allow",
-			conditionKey: "StringLike",
-			match:        true,
-		},
-		{
-			effect:       "Allow",
-			conditionKey: "StringNotLike",
-			match:        false,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "StringLike",
-			match:        true,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "StringNotLike",
-			match:        false,
-		},
-	}
-
-	for i, test := range testCases {
-		conditions := make(policy.ConditionMap)
-		conditions[test.conditionKey] = conditionsKeyMap
-
-		allowStatement := policy.Statement{
-			Sid:    "Testing AWS referer condition",
-			Effect: test.effect,
-			Principal: policy.User{
-				AWS: set.CreateStringSet("*"),
-			},
-			Resources:  resource,
-			Conditions: conditions,
-		}
-
-		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
-			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
-				i+1, test.match, result)
-		}
-	}
-}
-
-func TestAWSSourceIPCondition(t *testing.T) {
-	resource := set.CreateStringSet([]string{
-		fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/"+"Asia"+"*"),
-	}...)
-
-	conditionsKeyMap := make(policy.ConditionKeyMap)
-	// Test both IPv4 and IPv6 addresses.
-	conditionsKeyMap.Add("aws:SourceIp",
-		set.CreateStringSet("54.240.143.0/24",
-			"2001:DB8:1234:5678::/64"))
-
-	requestConditionMap := make(policy.ConditionKeyMap)
-	requestConditionMap["ip"] = set.CreateStringSet("54.240.143.2")
-
-	testCases := []struct {
-		effect       string
-		conditionKey string
-		match        bool
-	}{
-		{
-			effect:       "Allow",
-			conditionKey: "IpAddress",
-			match:        true,
-		},
-		{
-			effect:       "Allow",
-			conditionKey: "NotIpAddress",
-			match:        false,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "IpAddress",
-			match:        true,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "NotIpAddress",
-			match:        false,
-		},
-	}
-
-	for i, test := range testCases {
-		conditions := make(policy.ConditionMap)
-		conditions[test.conditionKey] = conditionsKeyMap
-
-		allowStatement := policy.Statement{
-			Sid:    "Testing AWS referer condition",
-			Effect: test.effect,
-			Principal: policy.User{
-				AWS: set.CreateStringSet("*"),
-			},
-			Resources:  resource,
-			Conditions: conditions,
-		}
-
-		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
-			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
-				i+1, test.match, result)
-		}
-	}
-}

cmd/bucket-policy.go

@@ -1,207 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"reflect"
-	"sync"
-
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio/pkg/errors"
-	"github.com/minio/minio/pkg/hash"
-)
-
-const (
-	// Static prefix to be used while constructing bucket ARN.
-	// refer to S3 docs for more info.
-	bucketARNPrefix = "arn:aws:s3:::"
-
-	// Bucket policy config name.
-	bucketPolicyConfig = "policy.json"
-)
-
-// Global bucket policies list, policies are enforced on each bucket looking
-// through the policies here.
-type bucketPolicies struct {
-	rwMutex *sync.RWMutex
-
-	// Collection of 'bucket' policies.
-	bucketPolicyConfigs map[string]policy.BucketAccessPolicy
-}
-
-// Fetch bucket policy for a given bucket.
-func (bp bucketPolicies) GetBucketPolicy(bucket string) policy.BucketAccessPolicy {
-	bp.rwMutex.RLock()
-	defer bp.rwMutex.RUnlock()
-	return bp.bucketPolicyConfigs[bucket]
-}
-
-// Set a new bucket policy for a bucket, this operation will overwrite
-// any previous bucket policies for the bucket.
-func (bp *bucketPolicies) SetBucketPolicy(bucket string, newpolicy policy.BucketAccessPolicy) error {
-	bp.rwMutex.Lock()
-	defer bp.rwMutex.Unlock()
-
-	if reflect.DeepEqual(newpolicy, emptyBucketPolicy) {
-		return errInvalidArgument
-	}
-	bp.bucketPolicyConfigs[bucket] = newpolicy
-
-	return nil
-}
-
-// Delete bucket policy from struct for a given bucket.
-func (bp *bucketPolicies) DeleteBucketPolicy(bucket string) error {
-	bp.rwMutex.Lock()
-	defer bp.rwMutex.Unlock()
-	delete(bp.bucketPolicyConfigs, bucket)
-	return nil
-}
-
-// Intialize all bucket policies.
-func initBucketPolicies(objAPI ObjectLayer) (*bucketPolicies, error) {
-	if objAPI == nil {
-		return nil, errInvalidArgument
-	}
-
-	// List buckets to proceed loading all notification configuration.
-	buckets, err := objAPI.ListBuckets(context.Background())
-	if err != nil {
-		return nil, errors.Cause(err)
-	}
-
-	policies := make(map[string]policy.BucketAccessPolicy)
-	// Loads bucket policy.
-	for _, bucket := range buckets {
-		bp, pErr := ReadBucketPolicy(bucket.Name, objAPI)
-		if pErr != nil {
-			// net.Dial fails for rpc client or any
-			// other unexpected errors during net.Dial.
-			if !errors.IsErrIgnored(pErr, errDiskNotFound) {
-				if !isErrBucketPolicyNotFound(pErr) {
-					return nil, errors.Cause(pErr)
-				}
-			}
-			// Continue to load other bucket policies if possible.
-			continue
-		}
-		policies[bucket.Name] = bp
-	}
-
-	// Return all bucket policies.
-	return &bucketPolicies{
-		rwMutex:             &sync.RWMutex{},
-		bucketPolicyConfigs: policies,
-	}, nil
-}
-
-// readBucketPolicyJSON - reads bucket policy for an input bucket, returns BucketPolicyNotFound
-// if bucket policy is not found.
-func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader io.Reader, err error) {
-	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-
-	var buffer bytes.Buffer
-	err = objAPI.GetObject(context.Background(), minioMetaBucket, policyPath, 0, -1, &buffer, "")
-	if err != nil {
-		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, PolicyNotFound{Bucket: bucket}
-		}
-		errorIf(err, "Unable to load policy for the bucket %s.", bucket)
-		return nil, errors.Cause(err)
-	}
-
-	return &buffer, nil
-}
-
-// ReadBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
-// if bucket policy is not found. This function also parses the bucket policy into an object.
-func ReadBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPolicy, error) {
-	// Read bucket policy JSON.
-	bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI)
-	if err != nil {
-		return emptyBucketPolicy, err
-	}
-
-	// Parse the saved policy.
-	var bp policy.BucketAccessPolicy
-	if err = parseBucketPolicy(bucketPolicyReader, &bp); err != nil {
-		return emptyBucketPolicy, err
-
-	}
-	return bp, nil
-}
-
-// removeBucketPolicy - removes any previously written bucket policy. Returns BucketPolicyNotFound
-// if no policies are found.
-func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
-	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-	err := objAPI.DeleteObject(context.Background(), minioMetaBucket, policyPath)
-	if err != nil {
-		err = errors.Cause(err)
-		if _, ok := err.(ObjectNotFound); ok {
-			return BucketPolicyNotFound{Bucket: bucket}
-		}
-		return err
-	}
-	return nil
-}
-
-// writeBucketPolicy - save a bucket policy that is assumed to be validated.
-func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy policy.BucketAccessPolicy) error {
-	buf, err := json.Marshal(bpy)
-	if err != nil {
-		errorIf(err, "Unable to marshal bucket policy '%#v' to JSON", bpy)
-		return err
-	}
-	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
-	if err != nil {
-		errorIf(err, "Unable to set policy for the bucket %s", bucket)
-		return errors.Cause(err)
-	}
-
-	if _, err = objAPI.PutObject(context.Background(), minioMetaBucket, policyPath, hashReader, nil); err != nil {
-		errorIf(err, "Unable to set policy for the bucket %s", bucket)
-		return errors.Cause(err)
-	}
-	return nil
-}
-
-// persistAndNotifyBucketPolicyChange - takes a policyChange argument,
-// persists it to storage, and notify nodes in the cluster about the
-// change. In-memory state is updated in response to the notification.
-func persistAndNotifyBucketPolicyChange(bucket string, isRemove bool, bktPolicy policy.BucketAccessPolicy, objAPI ObjectLayer) error {
-	if isRemove {
-		err := removeBucketPolicy(bucket, objAPI)
-		if err != nil {
-			return err
-		}
-	} else {
-		if reflect.DeepEqual(bktPolicy, emptyBucketPolicy) {
-			return errInvalidArgument
-		}
-		if err := writeBucketPolicy(bucket, objAPI, bktPolicy); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

cmd/build-constants.go

@@ -16,13 +16,14 @@
 
 package cmd
 
-import "go/build"
-
 // DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
 // set through ‘buildscripts/gen-ldflags.go’.
 var (
 	// GOPATH - GOPATH value at the time of build.
-	GOPATH = build.Default.GOPATH
+	GOPATH = ""
+
+	// GOROOT - GOROOT value at the time of build.
+	GOROOT = ""
 
 	// Go get development tag.
 	goGetTag = "DEVELOPMENT.GOGET"

cmd/certs.go

@@ -17,12 +17,16 @@
 package cmd
 
 import (
+	"bytes"
+	"crypto"
+	"crypto/ecdsa"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
-	"fmt"
 	"io/ioutil"
 	"os"
+
+	"github.com/minio/minio/pkg/certs"
 )
 
 // TLSPrivateKeyPassword is the environment variable which contains the password used
@@ -37,24 +41,27 @@ func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 		return nil, err
 	}
 
+	// Trimming leading and tailing white spaces.
+	data = bytes.TrimSpace(data)
+
 	// Parse all certs in the chain.
 	current := data
 	for len(current) > 0 {
 		var pemBlock *pem.Block
 		if pemBlock, current = pem.Decode(current); pemBlock == nil {
-			return nil, fmt.Errorf("Could not read PEM block from file %s", certFile)
+			return nil, uiErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
 		}
 
 		var x509Cert *x509.Certificate
 		if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
-			return nil, err
+			return nil, uiErrSSLUnexpectedData(err)
 		}
 
 		x509Certs = append(x509Certs, x509Cert)
 	}
 
 	if len(x509Certs) == 0 {
-		return nil, fmt.Errorf("Empty public certificate file %s", certFile)
+		return nil, uiErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
 	}
 
 	return x509Certs, nil
@@ -79,10 +86,11 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 		return nil, nil
 	}
 
-	rootCAs, err := x509.SystemCertPool()
-	if err != nil {
-		// In some systems like Windows, system cert pool is not supported.
-		// Hence we create a new cert pool.
+	rootCAs, _ := x509.SystemCertPool()
+	if rootCAs == nil {
+		// In some systems (like Windows) system cert pool is
+		// not supported or no certificates are present on the
+		// system - so we create a new cert pool.
 		rootCAs = x509.NewCertPool()
 	}
 
@@ -105,31 +113,44 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	certPEMBlock, err := ioutil.ReadFile(certFile)
 	if err != nil {
-		return tls.Certificate{}, fmt.Errorf("TLS: failed to read cert file: %v", err)
+		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
 	}
 	keyPEMBlock, err := ioutil.ReadFile(keyFile)
 	if err != nil {
-		return tls.Certificate{}, fmt.Errorf("TLS: failed to read private key: %v", err)
+		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
 	}
 	key, rest := pem.Decode(keyPEMBlock)
 	if len(rest) > 0 {
-		return tls.Certificate{}, fmt.Errorf("TLS: private key contains additional data")
+		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
 	}
 	if x509.IsEncryptedPEMBlock(key) {
 		password, ok := os.LookupEnv(TLSPrivateKeyPassword)
 		if !ok {
-			return tls.Certificate{}, fmt.Errorf("TLS: private key is encrypted but no password is present - set env var: %s", TLSPrivateKeyPassword)
+			return tls.Certificate{}, uiErrSSLNoPassword(nil)
 		}
 		decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
 		if decErr != nil {
-			return tls.Certificate{}, fmt.Errorf("TLS: failed to decrypt private key: %v", decErr)
+			return tls.Certificate{}, uiErrSSLWrongPassword(decErr)
 		}
 		keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
 	}
-	return tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+	cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+	if err != nil {
+		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg(err.Error())
+	}
+	// Ensure that the private key is not a P-384 or P-521 EC key.
+	// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
+	if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
+		if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
+			if name := pub.Params().Name; name == "P-384" || name == "P-521" { // unfortunately there is no cleaner way to check
+				return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
+			}
+		}
+	}
+	return cert, nil
 }
 
-func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
+func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, c *certs.Certs, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
 		return nil, nil, nil, false, nil
 	}
@@ -138,17 +159,15 @@ func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsC
 		return nil, nil, nil, false, err
 	}
 
-	var cert tls.Certificate
-	if cert, err = loadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
+	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
+	if err != nil {
 		return nil, nil, nil, false, err
 	}
 
-	tlsCert = &cert
-
 	if rootCAs, err = getRootCAs(getCADir()); err != nil {
 		return nil, nil, nil, false, err
 	}
 
 	secureConn = true
-	return x509Certs, rootCAs, tlsCert, secureConn, nil
+	return x509Certs, rootCAs, c, secureConn, nil
 }

cmd/common-main.go

@@ -17,15 +17,23 @@
 package cmd
 
 import (
+	"context"
 	"errors"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
+	etcd "github.com/coreos/etcd/client"
+
 	"github.com/minio/cli"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
 // Check for updates and print a notification message
@@ -33,21 +41,41 @@ func checkUpdate(mode string) {
 	// Its OK to ignore any errors during doUpdate() here.
 	if updateMsg, _, currentReleaseTime, latestReleaseTime, err := getUpdateInfo(2*time.Second, mode); err == nil {
 		if globalInplaceUpdateDisabled {
-			log.Println(updateMsg)
+			logger.StartupMessage(updateMsg)
 		} else {
-			log.Println(prepareUpdateMessage("Run `minio update`", latestReleaseTime.Sub(currentReleaseTime)))
+			logger.StartupMessage(prepareUpdateMessage("Run `minio update`", latestReleaseTime.Sub(currentReleaseTime)))
 		}
 	}
 }
 
+// Initialize and load config from remote etcd or local config directory
 func initConfig() {
-	// Config file does not exist, we create it fresh and return upon success.
+	if globalEtcdClient != nil {
+		kapi := etcd.NewKeysAPI(globalEtcdClient)
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		_, err := kapi.Get(ctx, getConfigFile(), nil)
+		cancel()
+		if err == nil {
+			logger.FatalIf(migrateConfig(), "Config migration failed.")
+			logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
+		} else {
+			if etcd.IsKeyNotFound(err) {
+				logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
+				logger.Info("Created minio configuration file successfully at %v", globalEtcdClient.Endpoints())
+			} else {
+				logger.FatalIf(err, "Unable to load config version: '%s'.", serverConfigVersion)
+			}
+		}
+		return
+	}
+
 	if isFile(getConfigFile()) {
-		fatalIf(migrateConfig(), "Config migration failed.")
-		fatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
+		logger.FatalIf(migrateConfig(), "Config migration failed")
+		logger.FatalIf(loadConfig(), "Unable to load the configuration file")
 	} else {
-		fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
-		log.Println("Created minio configuration file successfully at " + getConfigDir())
+		// Config file does not exist, we create it fresh and return upon success.
+		logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time")
+		logger.Info("Created minio configuration file successfully at " + getConfigDir())
 	}
 }
 
@@ -70,17 +98,17 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 		// default config directory.
 		configDir = getConfigDir()
 		if configDir == "" {
-			fatalIf(errors.New("missing option"), "config-dir option must be provided.")
+			logger.FatalIf(errors.New("missing option"), "config-dir option must be provided")
 		}
 	}
 
 	if configDir == "" {
-		fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
+		logger.FatalIf(errors.New("empty directory"), "Configuration directory cannot be empty")
 	}
 
 	// Disallow relative paths, figure out absolute paths.
 	configDirAbs, err := filepath.Abs(configDir)
-	fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
+	logger.FatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
 	setConfigDir(configDirAbs)
 }
 
@@ -94,7 +122,9 @@ func handleCommonEnvVars() {
 	secretKey := os.Getenv("MINIO_SECRET_KEY")
 	if accessKey != "" && secretKey != "" {
 		cred, err := auth.CreateCredentials(accessKey, secretKey)
-		fatalIf(err, "Invalid access/secret Key set in environment.")
+		if err != nil {
+			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
+		}
 
 		// credential Envs are set globally.
 		globalIsEnvCreds = true
@@ -102,9 +132,9 @@ func handleCommonEnvVars() {
 	}
 
 	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
-		browserFlag, err := ParseBrowserFlag(browser)
+		browserFlag, err := ParseBoolFlag(browser)
 		if err != nil {
-			fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
+			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Unable to validate MINIO_BROWSER environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -117,31 +147,74 @@ func handleCommonEnvVars() {
 	if traceFile != "" {
 		var err error
 		globalHTTPTraceFile, err = os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
-		fatalIf(err, "error opening file %s", traceFile)
+		logger.FatalIf(err, "error opening file %s", traceFile)
 	}
 
-	globalDomainName = os.Getenv("MINIO_DOMAIN")
-	if globalDomainName != "" {
-		globalIsEnvDomainName = true
+	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	if ok {
+		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
+		var err error
+		globalEtcdClient, err = etcd.New(etcd.Config{
+			Endpoints: etcdEndpoints,
+			Transport: NewCustomHTTPTransport(),
+		})
+		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+	}
+
+	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
+
+	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if ok {
+		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
+		globalDomainIPs = set.NewStringSet()
+		for i, ip := range minioEndpoints {
+			if net.ParseIP(ip) == nil {
+				logger.FatalIf(errInvalidArgument, "Unable to initialize Minio server with invalid MINIO_PUBLIC_IPS[%d]: %s", i, ip)
+			}
+			globalDomainIPs.Add(ip)
+		}
+	}
+	if globalDomainName != "" && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+		var err error
+		globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIPs, globalMinioPort, globalEtcdClient)
+		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainName)
 	}
 
 	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
 		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
-		fatalIf(err, "Invalid value set in environment variable MINIO_CACHE_DRIVES %s.", drives)
+		if err != nil {
+			logger.Fatal(err, "Unable to parse MINIO_CACHE_DRIVES value (%s)", drives)
+		}
 		globalCacheDrives = driveList
 		globalIsDiskCacheEnabled = true
 	}
+
 	if excludes := os.Getenv("MINIO_CACHE_EXCLUDE"); excludes != "" {
 		excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter))
-		fatalIf(err, "Invalid value set in environment variable MINIO_CACHE_EXCLUDE %s.", excludes)
+		if err != nil {
+			logger.Fatal(err, "Unable to parse MINIO_CACHE_EXCLUDE value (`%s`)", excludes)
+		}
 		globalCacheExcludes = excludeList
 	}
+
 	if expiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); expiryStr != "" {
 		expiry, err := strconv.Atoi(expiryStr)
-		fatalIf(err, "Invalid value set in environment variable MINIO_CACHE_EXPIRY %s.", expiryStr)
+		if err != nil {
+			logger.Fatal(uiErrInvalidCacheExpiryValue(err), "Unable to parse MINIO_CACHE_EXPIRY value (`%s`)", expiryStr)
+		}
 		globalCacheExpiry = expiry
 	}
 
+	if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" {
+		maxUse, err := strconv.Atoi(maxUseStr)
+		if err != nil {
+			logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr)
+		}
+		// maxUse should be a valid percentage.
+		if maxUse > 0 && maxUse <= 100 {
+			globalCacheMaxUse = maxUse
+		}
+	}
 	// In place update is true by default if the MINIO_UPDATE is not set
 	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
 	// in-place update is off.
@@ -154,29 +227,39 @@ func handleCommonEnvVars() {
 		// Check for environment variables and parse into storageClass struct
 		if ssc := os.Getenv(standardStorageClassEnv); ssc != "" {
 			globalStandardStorageClass, err = parseStorageClass(ssc)
-			fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
 		}
 
 		if rrsc := os.Getenv(reducedRedundancyStorageClassEnv); rrsc != "" {
 			globalRRStorageClass, err = parseStorageClass(rrsc)
-			fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
 		}
 
 		// Validation is done after parsing both the storage classes. This is needed because we need one
 		// storage class value to deduce the correct value of the other storage class.
 		if globalRRStorageClass.Scheme != "" {
 			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
 			globalIsStorageClass = true
 		}
 
 		if globalStandardStorageClass.Scheme != "" {
 			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
 			globalIsStorageClass = true
 		}
 	}
 
 	// Get WORM environment variable.
-	globalWORMEnabled = strings.EqualFold(os.Getenv("MINIO_WORM"), "on")
+	if worm := os.Getenv("MINIO_WORM"); worm != "" {
+		wormFlag, err := ParseBoolFlag(worm)
+		if err != nil {
+			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Unable to validate MINIO_WORM environment variable")
+		}
+
+		// worm Envs are set globally, this does not represent
+		// if worm is turned off or on.
+		globalIsEnvWORM = true
+		globalWORMEnabled = bool(wormFlag)
+	}
 }

cmd/config-current.go

@@ -19,15 +19,15 @@ package cmd
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"reflect"
 	"sync"
 
+	"github.com/minio/minio/cmd/logger"
+
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
 	"github.com/minio/minio/pkg/quick"
-	"github.com/tidwall/gjson"
 )
 
 // Steps to move from version N to version N+1
@@ -39,9 +39,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "23"
+const serverConfigVersion = "26"
 
-type serverConfig = serverConfigV23
+type serverConfig = serverConfigV26
 
 var (
 	// globalServerConfig server config.
@@ -85,7 +85,13 @@ func (s *serverConfig) GetCredential() auth.Credentials {
 // SetBrowser set if browser is enabled.
 func (s *serverConfig) SetBrowser(b bool) {
 	// Set the new value.
-	s.Browser = BrowserFlag(b)
+	s.Browser = BoolFlag(b)
+}
+
+// SetWorm set if worm is enabled.
+func (s *serverConfig) SetWorm(b bool) {
+	// Set the new value.
+	s.Worm = BoolFlag(b)
 }
 
 func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
@@ -99,16 +105,22 @@ func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
 	return s.StorageClass.Standard, s.StorageClass.RRS
 }
 
-// GetCredentials get current credentials.
+// GetBrowser get current credentials.
 func (s *serverConfig) GetBrowser() bool {
 	return bool(s.Browser)
 }
 
+// GetWorm get current credentials.
+func (s *serverConfig) GetWorm() bool {
+	return bool(s.Worm)
+}
+
 // SetCacheConfig sets the current cache config
-func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int) {
+func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxuse int) {
 	s.Cache.Drives = drives
 	s.Cache.Exclude = exclude
 	s.Cache.Expiry = expiry
+	s.Cache.MaxUse = maxuse
 }
 
 // GetCacheConfig gets the current cache config
@@ -116,10 +128,19 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
 	return s.Cache
 }
 
-// Save config.
-func (s *serverConfig) Save() error {
-	// Save config file.
-	return quick.Save(getConfigFile(), s)
+// Save config file to corresponding backend
+func Save(configFile string, data interface{}) error {
+	return quick.SaveConfig(data, configFile, globalEtcdClient)
+}
+
+// Load config from backend
+func Load(configFile string, data interface{}) (quick.Config, error) {
+	return quick.LoadConfig(configFile, globalEtcdClient, data)
+}
+
+// GetVersion gets config version from backend
+func GetVersion(configFile string) (string, error) {
+	return quick.GetVersion(configFile, globalEtcdClient)
 }
 
 // Returns the string describing a difference with the given
@@ -139,6 +160,8 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "Domain configuration differs"
 	case s.StorageClass != t.StorageClass:
 		return "StorageClass configuration differs"
+	case !reflect.DeepEqual(s.Cache, t.Cache):
+		return "Cache configuration differs"
 	case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
 		return "AMQP Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
@@ -167,9 +190,12 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 }
 
 func newServerConfig() *serverConfig {
+	cred, err := auth.GetNewCredentials()
+	logger.FatalIf(err, "")
+
 	srvCfg := &serverConfig{
 		Version:    serverConfigVersion,
-		Credential: auth.MustGetNewCredentials(),
+		Credential: cred,
 		Region:     globalMinioDefaultRegion,
 		Browser:    true,
 		StorageClass: storageClassConfig{
@@ -180,6 +206,7 @@ func newServerConfig() *serverConfig {
 			Drives:  []string{},
 			Exclude: []string{},
 			Expiry:  globalCacheExpiry,
+			MaxUse:  globalCacheMaxUse,
 		},
 		Notify: notifier{},
 	}
@@ -207,6 +234,7 @@ func newServerConfig() *serverConfig {
 	srvCfg.Cache.Drives = make([]string, 0)
 	srvCfg.Cache.Exclude = make([]string, 0)
 	srvCfg.Cache.Expiry = globalCacheExpiry
+	srvCfg.Cache.MaxUse = globalCacheMaxUse
 	return srvCfg
 }
 
@@ -214,135 +242,7 @@ func newServerConfig() *serverConfig {
 // found, otherwise use default parameters
 func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfig()
-
-	// If env is set override the credentials from config file.
-	if globalIsEnvCreds {
-		srvCfg.SetCredential(globalActiveCred)
-	}
-
-	if globalIsEnvBrowser {
-		srvCfg.SetBrowser(globalIsBrowserEnabled)
-	}
-
-	if globalIsEnvRegion {
-		srvCfg.SetRegion(globalServerRegion)
-	}
-
-	if globalIsEnvDomainName {
-		srvCfg.Domain = globalDomainName
-	}
-
-	if globalIsStorageClass {
-		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
-	}
-
-	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
-	}
-
-	// hold the mutex lock before a new config is assigned.
-	// Save the new config globally.
-	// unlock the mutex.
-	globalServerConfigMu.Lock()
-	globalServerConfig = srvCfg
-	globalServerConfigMu.Unlock()
-
-	// Save config into file.
-	return globalServerConfig.Save()
-}
-
-// doCheckDupJSONKeys recursively detects duplicate json keys
-func doCheckDupJSONKeys(key, value gjson.Result) error {
-	// Key occurrences map of the current scope to count
-	// if there is any duplicated json key.
-	keysOcc := make(map[string]int)
-
-	// Holds the found error
-	var checkErr error
-
-	// Iterate over keys in the current json scope
-	value.ForEach(func(k, v gjson.Result) bool {
-		// If current key is not null, check if its
-		// value contains some duplicated keys.
-		if k.Type != gjson.Null {
-			keysOcc[k.String()]++
-			checkErr = doCheckDupJSONKeys(k, v)
-		}
-		return checkErr == nil
-	})
-
-	// Check found err
-	if checkErr != nil {
-		return errors.New(key.String() + " => " + checkErr.Error())
-	}
-
-	// Check for duplicated keys
-	for k, v := range keysOcc {
-		if v > 1 {
-			return errors.New(key.String() + " => `" + k + "` entry is duplicated")
-		}
-	}
-
-	return nil
-}
-
-// Check recursively if a key is duplicated in the same json scope
-// e.g.:
-//  `{ "key" : { "key" ..` is accepted
-//  `{ "key" : { "subkey" : "val1", "subkey": "val2" ..` throws subkey duplicated error
-func checkDupJSONKeys(json string) error {
-	// Parse config with gjson library
-	config := gjson.Parse(json)
-
-	// Create a fake rootKey since root json doesn't seem to have representation
-	// in gjson library.
-	rootKey := gjson.Result{Type: gjson.String, Str: minioConfigFile}
-
-	// Check if loaded json contains any duplicated keys
-	return doCheckDupJSONKeys(rootKey, config)
-}
-
-// getValidConfig - returns valid server configuration
-func getValidConfig() (*serverConfig, error) {
-	srvCfg := &serverConfig{
-		Region:  globalMinioDefaultRegion,
-		Browser: true,
-	}
-
-	configFile := getConfigFile()
-	if _, err := quick.Load(configFile, srvCfg); err != nil {
-		return nil, err
-	}
-
-	if srvCfg.Version != serverConfigVersion {
-		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
-	}
-
-	// Load config file json and check for duplication json keys
-	jsonBytes, err := ioutil.ReadFile(configFile)
-	if err != nil {
-		return nil, err
-	}
-	if err = checkDupJSONKeys(string(jsonBytes)); err != nil {
-		return nil, err
-	}
-
-	// Validate credential fields only when
-	// they are not set via the environment
-
-	// Error out if global is env credential is not set and config has invalid credential
-	if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
-		return nil, errors.New("invalid credential in config file " + configFile)
-	}
-
-	return srvCfg, nil
-}
-
-// loadConfig - loads a new config from disk, overrides params from env
-// if found and valid
-func loadConfig() error {
-	srvCfg, err := getValidConfig()
+	srvCfg, err := newQuickConfig(newServerConfig())
 	if err != nil {
 		return err
 	}
@@ -356,6 +256,10 @@ func loadConfig() error {
 		srvCfg.SetBrowser(globalIsBrowserEnabled)
 	}
 
+	if globalIsEnvWORM {
+		srvCfg.SetWorm(globalWORMEnabled)
+	}
+
 	if globalIsEnvRegion {
 		srvCfg.SetRegion(globalServerRegion)
 	}
@@ -369,7 +273,95 @@ func loadConfig() error {
 	}
 
 	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
+		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
+	}
+
+	// hold the mutex lock before a new config is assigned.
+	// Save the new config globally.
+	// unlock the mutex.
+	globalServerConfigMu.Lock()
+	globalServerConfig = srvCfg
+	globalServerConfigMu.Unlock()
+
+	// Save config into file.
+	return Save(getConfigFile(), globalServerConfig)
+}
+
+// newQuickConfig - initialize a new server config, with an allocated
+// quick.Config interface.
+func newQuickConfig(srvCfg *serverConfig) (*serverConfig, error) {
+	qcfg, err := quick.NewConfig(srvCfg, globalEtcdClient)
+	if err != nil {
+		return nil, err
+	}
+
+	srvCfg.Config = qcfg
+	return srvCfg, nil
+}
+
+// getValidConfig - returns valid server configuration
+func getValidConfig() (*serverConfig, error) {
+	srvCfg := &serverConfig{
+		Region:  globalMinioDefaultRegion,
+		Browser: true,
+	}
+
+	var err error
+	srvCfg, err = newQuickConfig(srvCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	configFile := getConfigFile()
+	if err = srvCfg.Load(configFile); err != nil {
+		return nil, err
+	}
+
+	if srvCfg.Version != serverConfigVersion {
+		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
+	}
+
+	// Validate credential fields only when
+	// they are not set via the environment
+	// Error out if global is env credential is not set and config has invalid credential
+	if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
+		return nil, errors.New("invalid credential in config file " + getConfigFile())
+	}
+
+	return srvCfg, nil
+}
+
+// loadConfig - loads a new config from disk, overrides params from env
+// if found and valid
+func loadConfig() error {
+	srvCfg, err := getValidConfig()
+	if err != nil {
+		return uiErrInvalidConfig(nil).Msg(err.Error())
+	}
+
+	// If env is set override the credentials from config file.
+	if globalIsEnvCreds {
+		srvCfg.SetCredential(globalActiveCred)
+	}
+
+	if globalIsEnvBrowser {
+		srvCfg.SetBrowser(globalIsBrowserEnabled)
+	}
+
+	if globalIsEnvRegion {
+		srvCfg.SetRegion(globalServerRegion)
+	}
+
+	if globalIsEnvDomainName {
+		srvCfg.Domain = globalDomainName
+	}
+
+	if globalIsStorageClass {
+		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
+	}
+
+	if globalIsDiskCacheEnabled {
+		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
 	}
 
 	// hold the mutex lock before a new config is assigned.
@@ -381,6 +373,9 @@ func loadConfig() error {
 	if !globalIsEnvBrowser {
 		globalIsBrowserEnabled = globalServerConfig.GetBrowser()
 	}
+	if !globalIsEnvWORM {
+		globalWORMEnabled = globalServerConfig.GetWorm()
+	}
 	if !globalIsEnvRegion {
 		globalServerRegion = globalServerConfig.GetRegion()
 	}
@@ -395,6 +390,7 @@ func loadConfig() error {
 		globalCacheDrives = cacheConf.Drives
 		globalCacheExcludes = cacheConf.Exclude
 		globalCacheExpiry = cacheConf.Expiry
+		globalCacheMaxUse = cacheConf.MaxUse
 	}
 	globalServerConfigMu.Unlock()
 

cmd/config-current_test.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
-	"github.com/tidwall/gjson"
 )
 
 func TestServerConfig(t *testing.T) {
@@ -51,7 +50,7 @@ func TestServerConfig(t *testing.T) {
 	}
 
 	// Attempt to save.
-	if err := globalServerConfig.Save(); err != nil {
+	if err := globalServerConfig.Save(getConfigFile()); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 
@@ -126,34 +125,6 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	}
 }
 
-func TestCheckDupJSONKeys(t *testing.T) {
-	testCases := []struct {
-		json       string
-		shouldPass bool
-	}{
-		{`{}`, true},
-		{`{"version" : "13"}`, true},
-		{`{"version" : "13", "version": "14"}`, false},
-		{`{"version" : "13", "credential": {"accessKey": "12345"}}`, true},
-		{`{"version" : "13", "credential": {"accessKey": "12345", "accessKey":"12345"}}`, false},
-		{`{"version" : "13", "notify": {"amqp": {"1"}, "webhook":{"3"}}}`, true},
-		{`{"version" : "13", "notify": {"amqp": {"1"}, "amqp":{"3"}}}`, false},
-		{`{"version" : "13", "notify": {"amqp": {"1":{}, "2":{}}}}`, true},
-		{`{"version" : "13", "notify": {"amqp": {"1":{}, "1":{}}}}`, false},
-	}
-
-	for i, testCase := range testCases {
-		err := doCheckDupJSONKeys(gjson.Result{}, gjson.Parse(testCase.json))
-		if testCase.shouldPass && err != nil {
-			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, err)
-		}
-		if !testCase.shouldPass && err == nil {
-			t.Errorf("Test %d, should fail but it succeed.", i+1)
-		}
-	}
-
-}
-
 // Tests config validator..
 func TestValidateConfig(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)

cmd/config-migrate.go

@@ -21,6 +21,8 @@ import (
 	"os"
 	"path/filepath"
 
+	etcd "github.com/coreos/etcd/client"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
@@ -29,7 +31,7 @@ import (
 )
 
 // DO NOT EDIT following message template, please open a github issue to discuss instead.
-var configMigrateMSGTemplate = "Configuration file %s migrated from version '%s' to '%s' successfully.\n"
+var configMigrateMSGTemplate = "Configuration file %s migrated from version '%s' to '%s' successfully."
 
 // Migrates all config versions from "1" to "18".
 func migrateConfig() error {
@@ -41,7 +43,7 @@ func migrateConfig() error {
 	}
 
 	// Load only config version information.
-	version, err := quick.GetVersion(getConfigFile())
+	version, err := GetVersion(getConfigFile())
 	if err != nil {
 		return err
 	}
@@ -171,6 +173,21 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
+	case "23":
+		if err = migrateV23ToV24(); err != nil {
+			return err
+		}
+		fallthrough
+	case "24":
+		if err = migrateV24ToV25(); err != nil {
+			return err
+		}
+		fallthrough
+	case "25":
+		if err = migrateV25ToV26(); err != nil {
+			return err
+		}
+		fallthrough
 	case serverConfigVersion:
 		// No migration needed. this always points to current version.
 		err = nil
@@ -183,8 +200,8 @@ func purgeV1() error {
 	configFile := filepath.Join(getConfigDir(), "fsUsers.json")
 
 	cv1 := &configV1{}
-	_, err := quick.Load(configFile, cv1)
-	if os.IsNotExist(err) {
+	_, err := Load(configFile, cv1)
+	if os.IsNotExist(err) || etcd.IsKeyNotFound(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
@@ -194,7 +211,7 @@ func purgeV1() error {
 	}
 
 	os.RemoveAll(configFile)
-	log.Println("Removed unsupported config version ‘1’.")
+	logger.Info("Removed unsupported config version ‘1’.")
 	return nil
 }
 
@@ -204,7 +221,7 @@ func migrateV2ToV3() error {
 	configFile := getConfigFile()
 
 	cv2 := &configV2{}
-	_, err := quick.Load(configFile, cv2)
+	_, err := Load(configFile, cv2)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -248,11 +265,11 @@ func migrateV2ToV3() error {
 	}
 	srvConfig.Logger.Syslog = slogger
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv2.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv2.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv2.Version, srvConfig.Version)
 	return nil
 }
 
@@ -263,7 +280,7 @@ func migrateV3ToV4() error {
 	configFile := getConfigFile()
 
 	cv3 := &configV3{}
-	_, err := quick.Load(configFile, cv3)
+	_, err := Load(configFile, cv3)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -286,11 +303,11 @@ func migrateV3ToV4() error {
 	srvConfig.Logger.File = cv3.Logger.File
 	srvConfig.Logger.Syslog = cv3.Logger.Syslog
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv3.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv3.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv3.Version, srvConfig.Version)
 	return nil
 }
 
@@ -301,7 +318,7 @@ func migrateV4ToV5() error {
 	configFile := getConfigFile()
 
 	cv4 := &configV4{}
-	_, err := quick.Load(configFile, cv4)
+	_, err := Load(configFile, cv4)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -327,11 +344,11 @@ func migrateV4ToV5() error {
 	srvConfig.Logger.ElasticSearch.Enable = false
 	srvConfig.Logger.Redis.Enable = false
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv4.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv4.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv4.Version, srvConfig.Version)
 	return nil
 }
 
@@ -342,7 +359,7 @@ func migrateV5ToV6() error {
 	configFile := getConfigFile()
 
 	cv5 := &configV5{}
-	_, err := quick.Load(configFile, cv5)
+	_, err := Load(configFile, cv5)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -416,11 +433,11 @@ func migrateV5ToV6() error {
 		}
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv5.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv5.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv5.Version, srvConfig.Version)
 	return nil
 }
 
@@ -431,7 +448,7 @@ func migrateV6ToV7() error {
 	configFile := getConfigFile()
 
 	cv6 := &configV6{}
-	_, err := quick.Load(configFile, cv6)
+	_, err := Load(configFile, cv6)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -472,11 +489,11 @@ func migrateV6ToV7() error {
 		srvConfig.Notify.Redis = cv6.Notify.Redis
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv6.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv6.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv6.Version, srvConfig.Version)
 	return nil
 }
 
@@ -487,7 +504,7 @@ func migrateV7ToV8() error {
 	configFile := getConfigFile()
 
 	cv7 := &serverConfigV7{}
-	_, err := quick.Load(configFile, cv7)
+	_, err := Load(configFile, cv7)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -535,11 +552,11 @@ func migrateV7ToV8() error {
 		srvConfig.Notify.Redis = cv7.Notify.Redis
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv7.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv7.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv7.Version, srvConfig.Version)
 	return nil
 }
 
@@ -549,7 +566,7 @@ func migrateV8ToV9() error {
 	configFile := getConfigFile()
 
 	cv8 := &serverConfigV8{}
-	_, err := quick.Load(configFile, cv8)
+	_, err := Load(configFile, cv8)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -605,11 +622,11 @@ func migrateV8ToV9() error {
 		srvConfig.Notify.PostgreSQL = cv8.Notify.PostgreSQL
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv8.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv8.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv8.Version, srvConfig.Version)
 	return nil
 }
 
@@ -619,7 +636,7 @@ func migrateV9ToV10() error {
 	configFile := getConfigFile()
 
 	cv9 := &serverConfigV9{}
-	_, err := quick.Load(configFile, cv9)
+	_, err := Load(configFile, cv9)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -673,11 +690,11 @@ func migrateV9ToV10() error {
 		srvConfig.Notify.PostgreSQL = cv9.Notify.PostgreSQL
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv9.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv9.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv9.Version, srvConfig.Version)
 	return nil
 }
 
@@ -687,7 +704,7 @@ func migrateV10ToV11() error {
 	configFile := getConfigFile()
 
 	cv10 := &serverConfigV10{}
-	_, err := quick.Load(configFile, cv10)
+	_, err := Load(configFile, cv10)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -744,11 +761,11 @@ func migrateV10ToV11() error {
 	srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
 	srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv10.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv10.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv10.Version, srvConfig.Version)
 	return nil
 }
 
@@ -758,7 +775,7 @@ func migrateV11ToV12() error {
 	configFile := getConfigFile()
 
 	cv11 := &serverConfigV11{}
-	_, err := quick.Load(configFile, cv11)
+	_, err := Load(configFile, cv11)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -842,11 +859,11 @@ func migrateV11ToV12() error {
 		}
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv11.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv11.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv11.Version, srvConfig.Version)
 	return nil
 }
 
@@ -855,7 +872,7 @@ func migrateV12ToV13() error {
 	configFile := getConfigFile()
 
 	cv12 := &serverConfigV12{}
-	_, err := quick.Load(configFile, cv12)
+	_, err := Load(configFile, cv12)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -922,11 +939,11 @@ func migrateV12ToV13() error {
 	srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
 	srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv12.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv12.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv12.Version, srvConfig.Version)
 	return nil
 }
 
@@ -935,7 +952,7 @@ func migrateV13ToV14() error {
 	configFile := getConfigFile()
 
 	cv13 := &serverConfigV13{}
-	_, err := quick.Load(configFile, cv13)
+	_, err := Load(configFile, cv13)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1007,11 +1024,11 @@ func migrateV13ToV14() error {
 	// Set the new browser parameter to true by default
 	srvConfig.Browser = true
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv13.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv13.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv13.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1020,7 +1037,7 @@ func migrateV14ToV15() error {
 	configFile := getConfigFile()
 
 	cv14 := &serverConfigV14{}
-	_, err := quick.Load(configFile, cv14)
+	_, err := Load(configFile, cv14)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1096,11 +1113,11 @@ func migrateV14ToV15() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv14.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv14.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv14.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv14.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1110,7 +1127,7 @@ func migrateV15ToV16() error {
 	configFile := getConfigFile()
 
 	cv15 := &serverConfigV15{}
-	_, err := quick.Load(configFile, cv15)
+	_, err := Load(configFile, cv15)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1186,11 +1203,11 @@ func migrateV15ToV16() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv15.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv15.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv15.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1200,7 +1217,7 @@ func migrateV16ToV17() error {
 	configFile := getConfigFile()
 
 	cv16 := &serverConfigV16{}
-	_, err := quick.Load(configFile, cv16)
+	_, err := Load(configFile, cv16)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1307,11 +1324,11 @@ func migrateV16ToV17() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv16.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv16.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv16.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv16.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1321,7 +1338,7 @@ func migrateV17ToV18() error {
 	configFile := getConfigFile()
 
 	cv17 := &serverConfigV17{}
-	_, err := quick.Load(configFile, cv17)
+	_, err := Load(configFile, cv17)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1411,11 +1428,11 @@ func migrateV17ToV18() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv17.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv17.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv17.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv17.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1423,7 +1440,7 @@ func migrateV18ToV19() error {
 	configFile := getConfigFile()
 
 	cv18 := &serverConfigV18{}
-	_, err := quick.Load(configFile, cv18)
+	_, err := Load(configFile, cv18)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1517,11 +1534,11 @@ func migrateV18ToV19() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv18.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv18.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1529,7 +1546,7 @@ func migrateV19ToV20() error {
 	configFile := getConfigFile()
 
 	cv19 := &serverConfigV19{}
-	_, err := quick.Load(configFile, cv19)
+	_, err := Load(configFile, cv19)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1622,11 +1639,11 @@ func migrateV19ToV20() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv19.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1634,7 +1651,7 @@ func migrateV20ToV21() error {
 	configFile := getConfigFile()
 
 	cv20 := &serverConfigV20{}
-	_, err := quick.Load(configFile, cv20)
+	_, err := Load(configFile, cv20)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1726,11 +1743,11 @@ func migrateV20ToV21() error {
 	// Load domain config from existing config in the file.
 	srvConfig.Domain = cv20.Domain
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv20.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv20.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv20.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1738,7 +1755,7 @@ func migrateV21ToV22() error {
 	configFile := getConfigFile()
 
 	cv21 := &serverConfigV21{}
-	_, err := quick.Load(configFile, cv21)
+	_, err := Load(configFile, cv21)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1830,11 +1847,11 @@ func migrateV21ToV22() error {
 	// Load domain config from existing config in the file.
 	srvConfig.Domain = cv21.Domain
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv21.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv21.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv21.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1842,7 +1859,7 @@ func migrateV22ToV23() error {
 	configFile := getConfigFile()
 
 	cv22 := &serverConfigV22{}
-	_, err := quick.Load(configFile, cv22)
+	_, err := Load(configFile, cv22)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1943,10 +1960,360 @@ func migrateV22ToV23() error {
 	srvConfig.Cache.Exclude = []string{}
 	srvConfig.Cache.Expiry = globalCacheExpiry
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv22.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv22.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv22.Version, srvConfig.Version)
+	return nil
+}
+
+func migrateV23ToV24() error {
+	configFile := getConfigFile()
+
+	cv23 := &serverConfigV23{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv23)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘23’. %v", err)
+	}
+	if cv23.Version != "23" {
+		return nil
+	}
+
+	// Copy over fields from V23 into V24 config struct
+	srvConfig := &serverConfigV24{
+		Notify: notifier{},
+	}
+	srvConfig.Version = "24"
+	srvConfig.Credential = cv23.Credential
+	srvConfig.Region = cv23.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	if len(cv23.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]target.AMQPArgs)
+		srvConfig.Notify.AMQP["1"] = target.AMQPArgs{}
+	} else {
+		srvConfig.Notify.AMQP = cv23.Notify.AMQP
+	}
+	if len(cv23.Notify.Elasticsearch) == 0 {
+		srvConfig.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
+		srvConfig.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Elasticsearch = cv23.Notify.Elasticsearch
+	}
+	if len(cv23.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]target.RedisArgs)
+		srvConfig.Notify.Redis["1"] = target.RedisArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv23.Notify.Redis
+	}
+	if len(cv23.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
+		srvConfig.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv23.Notify.PostgreSQL
+	}
+	if len(cv23.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
+		srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
+	} else {
+		srvConfig.Notify.Kafka = cv23.Notify.Kafka
+	}
+	if len(cv23.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]target.NATSArgs)
+		srvConfig.Notify.NATS["1"] = target.NATSArgs{}
+	} else {
+		srvConfig.Notify.NATS = cv23.Notify.NATS
+	}
+	if len(cv23.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
+		srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
+	} else {
+		srvConfig.Notify.Webhook = cv23.Notify.Webhook
+	}
+	if len(cv23.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]target.MySQLArgs)
+		srvConfig.Notify.MySQL["1"] = target.MySQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv23.Notify.MySQL
+	}
+
+	if len(cv23.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]target.MQTTArgs)
+		srvConfig.Notify.MQTT["1"] = target.MQTTArgs{}
+	} else {
+		srvConfig.Notify.MQTT = cv23.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv23.Browser
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv23.Domain
+
+	// Load storage class config from existing storage class config in the file.
+	srvConfig.StorageClass.RRS = cv23.StorageClass.RRS
+	srvConfig.StorageClass.Standard = cv23.StorageClass.Standard
+
+	// Load cache config from existing cache config in the file.
+	srvConfig.Cache.Drives = cv23.Cache.Drives
+	srvConfig.Cache.Exclude = cv23.Cache.Exclude
+	srvConfig.Cache.Expiry = cv23.Cache.Expiry
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv23.Version, srvConfig.Version, err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, cv23.Version, srvConfig.Version)
+	return nil
+}
+
+func migrateV24ToV25() error {
+	configFile := getConfigFile()
+
+	cv24 := &serverConfigV24{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv24)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘24’. %v", err)
+	}
+	if cv24.Version != "24" {
+		return nil
+	}
+
+	// Copy over fields from V24 into V25 config struct
+	srvConfig := &serverConfigV25{
+		Notify: notifier{},
+	}
+	srvConfig.Version = "25"
+	srvConfig.Credential = cv24.Credential
+	srvConfig.Region = cv24.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	if len(cv24.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]target.AMQPArgs)
+		srvConfig.Notify.AMQP["1"] = target.AMQPArgs{}
+	} else {
+		srvConfig.Notify.AMQP = cv24.Notify.AMQP
+	}
+	if len(cv24.Notify.Elasticsearch) == 0 {
+		srvConfig.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
+		srvConfig.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Elasticsearch = cv24.Notify.Elasticsearch
+	}
+	if len(cv24.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]target.RedisArgs)
+		srvConfig.Notify.Redis["1"] = target.RedisArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv24.Notify.Redis
+	}
+	if len(cv24.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
+		srvConfig.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv24.Notify.PostgreSQL
+	}
+	if len(cv24.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
+		srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
+	} else {
+		srvConfig.Notify.Kafka = cv24.Notify.Kafka
+	}
+	if len(cv24.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]target.NATSArgs)
+		srvConfig.Notify.NATS["1"] = target.NATSArgs{}
+	} else {
+		srvConfig.Notify.NATS = cv24.Notify.NATS
+	}
+	if len(cv24.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
+		srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
+	} else {
+		srvConfig.Notify.Webhook = cv24.Notify.Webhook
+	}
+	if len(cv24.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]target.MySQLArgs)
+		srvConfig.Notify.MySQL["1"] = target.MySQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv24.Notify.MySQL
+	}
+
+	if len(cv24.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]target.MQTTArgs)
+		srvConfig.Notify.MQTT["1"] = target.MQTTArgs{}
+	} else {
+		srvConfig.Notify.MQTT = cv24.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv24.Browser
+
+	// New field should be turned-off by default.
+	srvConfig.Worm = false // cv25.Worm should be used here
+	// for the next migration from v25 to v26 to persist
+	// local config value.
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv24.Domain
+
+	// Load storage class config from existing storage class config in the file.
+	srvConfig.StorageClass.RRS = cv24.StorageClass.RRS
+	srvConfig.StorageClass.Standard = cv24.StorageClass.Standard
+
+	// Load cache config from existing cache config in the file.
+	srvConfig.Cache.Drives = cv24.Cache.Drives
+	srvConfig.Cache.Exclude = cv24.Cache.Exclude
+	srvConfig.Cache.Expiry = cv24.Cache.Expiry
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv24.Version, srvConfig.Version, err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, cv24.Version, srvConfig.Version)
+	return nil
+}
+
+func migrateV25ToV26() error {
+	configFile := getConfigFile()
+
+	cv25 := &serverConfigV25{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘25’. %v", err)
+	}
+	if cv25.Version != "25" {
+		return nil
+	}
+
+	// Copy over fields from V25 into V26 config struct
+	srvConfig := &serverConfigV26{
+		Notify: notifier{},
+	}
+	srvConfig.Version = "26"
+	srvConfig.Credential = cv25.Credential
+	srvConfig.Region = cv25.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	if len(cv25.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]target.AMQPArgs)
+		srvConfig.Notify.AMQP["1"] = target.AMQPArgs{}
+	} else {
+		srvConfig.Notify.AMQP = cv25.Notify.AMQP
+	}
+	if len(cv25.Notify.Elasticsearch) == 0 {
+		srvConfig.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
+		srvConfig.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Elasticsearch = cv25.Notify.Elasticsearch
+	}
+	if len(cv25.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]target.RedisArgs)
+		srvConfig.Notify.Redis["1"] = target.RedisArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv25.Notify.Redis
+	}
+	if len(cv25.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
+		srvConfig.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv25.Notify.PostgreSQL
+	}
+	if len(cv25.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
+		srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
+	} else {
+		srvConfig.Notify.Kafka = cv25.Notify.Kafka
+	}
+	if len(cv25.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]target.NATSArgs)
+		srvConfig.Notify.NATS["1"] = target.NATSArgs{}
+	} else {
+		srvConfig.Notify.NATS = cv25.Notify.NATS
+	}
+	if len(cv25.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
+		srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
+	} else {
+		srvConfig.Notify.Webhook = cv25.Notify.Webhook
+	}
+	if len(cv25.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]target.MySQLArgs)
+		srvConfig.Notify.MySQL["1"] = target.MySQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv25.Notify.MySQL
+	}
+
+	if len(cv25.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]target.MQTTArgs)
+		srvConfig.Notify.MQTT["1"] = target.MQTTArgs{}
+	} else {
+		srvConfig.Notify.MQTT = cv25.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv25.Browser
+
+	// Load worm config from existing config in the file.
+	srvConfig.Worm = cv25.Worm
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv25.Domain
+
+	// Load storage class config from existing storage class config in the file.
+	srvConfig.StorageClass.RRS = cv25.StorageClass.RRS
+	srvConfig.StorageClass.Standard = cv25.StorageClass.Standard
+
+	// Load cache config from existing cache config in the file.
+	srvConfig.Cache.Drives = cv25.Cache.Drives
+	srvConfig.Cache.Exclude = cv25.Cache.Exclude
+	srvConfig.Cache.Expiry = cv25.Cache.Expiry
+
+	// Add predefined value to new server config.
+	srvConfig.Cache.MaxUse = globalCacheMaxUse
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv25.Version, srvConfig.Version, err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, cv25.Version, srvConfig.Version)
 	return nil
 }

cmd/config-versions.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/quick"
 )
 
 /////////////////// Config V1 ///////////////////
@@ -405,7 +406,7 @@ type serverConfigV14 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -422,7 +423,7 @@ type serverConfigV15 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -460,7 +461,7 @@ type serverConfigV16 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -479,7 +480,7 @@ type serverConfigV17 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -498,7 +499,7 @@ type serverConfigV18 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -516,7 +517,7 @@ type serverConfigV19 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -534,7 +535,7 @@ type serverConfigV20 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Additional error logging configuration.
@@ -552,7 +553,7 @@ type serverConfigV21 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Notification queue configuration.
@@ -570,7 +571,7 @@ type serverConfigV22 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
@@ -580,14 +581,95 @@ type serverConfigV22 struct {
 	Notify notifier `json:"notify"`
 }
 
-// serverConfigV23 is just like version '22' with addition of cache field
+// serverConfigV23 is just like version '22' with addition of cache field.
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV23 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// serverConfigV24 is just like version '23', we had to revert
+// the changes which were made in 6fb06045028b7a57c37c60a612c8e50735279ab4
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV24 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// serverConfigV25 is just like version '24', stores additionally
+// worm variable.
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV25 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// serverConfigV26 is just like version '25', stores additionally
+// cache max use value in 'CacheConfig'.
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV26 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration

cmd/crossdomain-xml-handler_test.go

@@ -21,14 +21,14 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 )
 
 // Test cross domain xml handler.
 func TestCrossXMLHandler(t *testing.T) {
 	// Server initialization.
-	mux := router.NewRouter().SkipClean(true)
-	handler := setCrossDomainPolicy(mux)
+	router := mux.NewRouter().SkipClean(true)
+	handler := setCrossDomainPolicy(router)
 	srv := httptest.NewServer(handler)
 
 	resp, err := http.Get(srv.URL + crossDomainXMLEntity)

cmd/crypto/doc.go

@@ -0,0 +1,116 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package crypto implements AWS S3 related cryptographic building blocks
+// for implementing Server-Side-Encryption (SSE-S3) and Server-Side-Encryption
+// with customer provided keys (SSE-C).
+//
+// All objects are encrypted with an unique and randomly generated 'ObjectKey'.
+// The ObjectKey itself is never stored in plaintext. Instead it is only stored
+// in a sealed from. The sealed 'ObjectKey' is created by encrypting the 'ObjectKey'
+// with an unique key-encryption-key. Given the correct key-encryption-key the
+// sealed 'ObjectKey' can be unsealed and the object can be decrypted.
+//
+//
+// ## SSE-C
+//
+// SSE-C computes the key-encryption-key from the client-provided key, an
+// initialization vector (IV) and the bucket/object path.
+//
+// 1. Encrypt:
+//       Input: ClientKey, bucket, object, metadata, object_data
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: ClientKey, bucket, object, metadata, enc_object_data
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -   KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+//
+// ## SSE-S3
+//
+// SSE-S3 can use either a master key or a KMS as root-of-trust.
+// The en/decryption slightly depens upon which root-of-trust is used.
+//
+// ### SSE-S3 and single master key
+//
+// The master key is used to derive unique object- and key-encryption-keys.
+// SSE-S3 with a single master key works as SSE-C where the master key is
+// used as the client-provided key.
+//
+// 1. Encrypt:
+//       Input: MasterKey, bucket, object, metadata, object_data
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: MasterKey, bucket, object, metadata, enc_object_data
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -   KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+//
+// ### SSE-S3 and KMS
+//
+// SSE-S3 requires that the KMS provides two functions:
+//       1.       Generate(KeyID) -> (Key, EncKey)
+//       2. Unseal(KeyID, EncKey) -> Key
+//
+// 1. Encrypt:
+//       Input: KeyID, bucket, object, metadata, object_data
+//       -     Key, EncKey := Generate(KeyID)
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(Key, Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- KeyID
+//       -        metadata <- EncKey
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: bucket, object, metadata, enc_object_data
+//       -      KeyID  <- metadata
+//       -      EncKey <- metadata
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -         Key := Unseal(KeyID, EncKey)
+//       -   KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+package crypto

cmd/crypto/error.go

@@ -0,0 +1,23 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import "errors"
+
+var (
+	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
+	// is not supported.
+	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+)

cmd/crypto/header.go

@@ -0,0 +1,49 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"net/http"
+)
+
+// SSEHeader is the general AWS SSE HTTP header key.
+const SSEHeader = "X-Amz-Server-Side-Encryption"
+
+// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
+// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
+// For SSE-C  see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
+const SSEAlgorithmAES256 = "AES256"
+
+// S3 represents AWS SSE-S3. It provides functionality to handle
+// SSE-S3 requests.
+var S3 = s3{}
+
+type s3 struct{}
+
+// IsRequested returns true if the HTTP headers indicates that
+// the S3 client requests SSE-S3.
+func (s3) IsRequested(h http.Header) bool {
+	_, ok := h[SSEHeader]
+	return ok
+}
+
+// Parse parses the SSE-S3 related HTTP headers and checks
+// whether they contain valid values.
+func (s3) Parse(h http.Header) (err error) {
+	if h.Get(SSEHeader) != SSEAlgorithmAES256 {
+		err = ErrInvalidEncryptionMethod
+	}
+	return
+}

cmd/crypto/header_test.go

@@ -0,0 +1,56 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"net/http"
+	"testing"
+)
+
+var isRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},  // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},        // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},  // 3
+}
+
+func TestS3IsRequested(t *testing.T) {
+	for i, test := range isRequestedTests {
+		if got := S3.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var parseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, ExpectedErr: nil},                         // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, ExpectedErr: ErrInvalidEncryptionMethod},        // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, ExpectedErr: ErrInvalidEncryptionMethod},   // 3
+}
+
+func TestS3Parse(t *testing.T) {
+	for i, test := range parseTests {
+		if err := S3.Parse(test.Header); err != test.ExpectedErr {
+			t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+	}
+}

cmd/crypto/key.go

@@ -0,0 +1,93 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"io"
+	"path/filepath"
+
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+// ObjectKey is a 256 bit secret key used to encrypt the object.
+// It must never be stored in plaintext.
+type ObjectKey [32]byte
+
+// GenerateKey generates a unique ObjectKey from a 256 bit external key
+// and a source of randomness. If random is nil the default PRNG of system
+// (crypto/rand) is used.
+func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
+	if random == nil {
+		random = rand.Reader
+	}
+	var nonce [32]byte
+	if _, err := io.ReadFull(random, nonce[:]); err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
+	}
+	sha := sha256.New()
+	sha.Write(extKey[:])
+	sha.Write(nonce[:])
+	sha.Sum(key[:0])
+	return
+}
+
+// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
+// key is also cryptographically bound to the object's path (bucket/object).
+func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte {
+	var sealedKey bytes.Buffer
+	mac := hmac.New(sha256.New, extKey[:])
+	mac.Write(iv[:])
+	mac.Write([]byte(filepath.Join(bucket, object)))
+
+	if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
+	}
+	return sealedKey.Bytes()
+}
+
+// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key
+// is cryptographically bound to the object's path the same bucket/object as during sealing
+// must be provided. On success the ObjectKey contains the decrypted sealed key.
+func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error {
+	var unsealedKey bytes.Buffer
+	mac := hmac.New(sha256.New, extKey[:])
+	mac.Write(iv[:])
+	mac.Write([]byte(filepath.Join(bucket, object)))
+
+	if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil {
+		return err // TODO(aead): upgrade sio to use sio.Error
+	}
+	copy(key[:], unsealedKey.Bytes())
+	return nil
+}
+
+// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index.
+func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
+	var bin [4]byte
+	binary.LittleEndian.PutUint32(bin[:], id)
+
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write(bin[:])
+	mac.Sum(partKey[:0])
+	return
+}

cmd/crypto/key_test.go

@@ -0,0 +1,127 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"io"
+	"testing"
+)
+
+var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
+
+func recoverTest(i int, shouldPass bool, t *testing.T) {
+	if err := recover(); err == nil && !shouldPass {
+		t.Errorf("Test %d should fail but passed successfully", i)
+	} else if err != nil && shouldPass {
+		t.Errorf("Test %d should pass but failed: %v", i, err)
+	}
+}
+
+var generateKeyTests = []struct {
+	ExtKey     [32]byte
+	Random     io.Reader
+	ShouldPass bool
+}{
+	{ExtKey: [32]byte{}, Random: nil, ShouldPass: true},             // 0
+	{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true},     // 1
+	{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
+	// {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064
+}
+
+func TestGenerateKey(t *testing.T) {
+	for i, test := range generateKeyTests {
+		func() {
+			defer recoverTest(i, test.ShouldPass, t)
+			key := GenerateKey(test.ExtKey, test.Random)
+			if [32]byte(key) == [32]byte{} {
+				t.Errorf("Test %d: generated key is zero key", i) // check that we generate random and unique key
+			}
+		}()
+	}
+}
+
+var sealUnsealKeyTests = []struct {
+	SealExtKey, SealIV     [32]byte
+	SealBucket, SealObject string
+
+	UnsealExtKey, UnsealIV     [32]byte
+	UnsealBucket, UnsealObject string
+
+	ShouldPass bool
+}{
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: true,
+	}, // 0
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 1
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 2
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 3
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object",
+		ShouldPass: false,
+	}, // 4
+}
+
+func TestSealUnsealKey(t *testing.T) {
+	for i, test := range sealUnsealKeyTests {
+		key := GenerateKey(test.SealExtKey, rand.Reader)
+		sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject)
+		if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
+			t.Errorf("Test %d should fail but passed successfully", i)
+		} else if err != nil && test.ShouldPass {
+			t.Errorf("Test %d should pass put failed: %v", i, err)
+		}
+	}
+}
+
+var derivePartKeyTest = []struct {
+	PartID  uint32
+	PartKey string
+}{
+	{PartID: 0, PartKey: "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"},
+	{PartID: 1, PartKey: "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"},
+	{PartID: 10000, PartKey: "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"},
+}
+
+func TestDerivePartKey(t *testing.T) {
+	var key ObjectKey
+	for i, test := range derivePartKeyTest {
+		expectedPartKey, err := hex.DecodeString(test.PartKey)
+		if err != nil {
+			t.Fatalf("Test %d failed to decode expected part-key: %v", i, err)
+		}
+		partKey := key.DerivePartKey(test.PartID)
+		if !bytes.Equal(partKey[:], expectedPartKey[:]) {
+			t.Errorf("Test %d derives wrong part-key: got '%s' want: '%s'", i, hex.EncodeToString(partKey[:]), test.PartKey)
+		}
+	}
+}

cmd/crypto/sse.go

@@ -0,0 +1,60 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"context"
+	"errors"
+	"io"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/ioutil"
+	"github.com/minio/sio"
+)
+
+const (
+	// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
+	S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
+	// S3KMSKeyID is the metadata key referencing the KMS key-id used to
+	// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
+	S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
+	// S3KMSSealedKey is the metadata key referencing the encrypted key generated
+	// by KMS. It is only used for SSE-S3 + KMS.
+	S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
+)
+
+// EncryptSinglePart encrypts an io.Reader which must be the
+// the body of a single-part PUT request.
+func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
+	r, err := sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, Key: key[:]})
+	if err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to encrypt io.Reader using object key"))
+	}
+	return r
+}
+
+// DecryptSinglePart decrypts an io.Writer which must an object
+// uploaded with the single-part PUT API. The offset and length
+// specify the requested range.
+func DecryptSinglePart(w io.Writer, offset, length int64, key ObjectKey) io.WriteCloser {
+	const PayloadSize = 1 << 16 // DARE 2.0
+	w = ioutil.LimitedWriter(w, offset%PayloadSize, length)
+
+	decWriter, err := sio.DecryptWriter(w, sio.Config{Key: key[:]})
+	if err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to decrypt io.Writer using object key"))
+	}
+	return decWriter
+}

cmd/disk-cache-config.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"encoding/json"
-	"fmt"
 	"path/filepath"
 )
 
@@ -26,6 +25,7 @@ import (
 type CacheConfig struct {
 	Drives  []string `json:"drives"`
 	Expiry  int      `json:"expiry"`
+	MaxUse  int      `json:"maxuse"`
 	Exclude []string `json:"exclude"`
 }
 
@@ -54,7 +54,7 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
 func parseCacheDrives(drives []string) ([]string, error) {
 	for _, d := range drives {
 		if !filepath.IsAbs(d) {
-			return nil, fmt.Errorf("cache dir should be absolute path: %s", d)
+			return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
 		}
 	}
 	return drives, nil
@@ -64,10 +64,10 @@ func parseCacheDrives(drives []string) ([]string, error) {
 func parseCacheExcludes(excludes []string) ([]string, error) {
 	for _, e := range excludes {
 		if len(e) == 0 {
-			return nil, fmt.Errorf("cache exclude path (%s) cannot be empty", e)
+			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
 		}
 		if hasPrefix(e, slashSeparator) {
-			return nil, fmt.Errorf("cache exclude pattern (%s) cannot start with / as prefix", e)
+			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
 		}
 	}
 	return excludes, nil

cmd/disk-cache-fs.go

@@ -28,8 +28,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
-	errors2 "github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 )
@@ -37,7 +37,6 @@ import (
 const (
 	// cache.json object metadata for cached objects.
 	cacheMetaJSONFile = "cache.json"
-	cacheMetaFormat   = "cache"
 
 	cacheEnvDelimiter = ";"
 )
@@ -93,7 +92,7 @@ func newCacheFSObjects(dir string, expiry int, maxDiskUsagePct int) (*cacheFSObj
 		appendFileMap: make(map[string]*fsAppendFile),
 	}
 
-	go fsObjects.cleanupStaleMultipartUploads(globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
+	go fsObjects.cleanupStaleMultipartUploads(context.Background(), globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
 
 	cacheFS := cacheFSObjects{
 		FSObjects:       fsObjects,
@@ -116,7 +115,9 @@ func (cfs *cacheFSObjects) diskUsageLow() bool {
 	minUsage := cfs.maxDiskUsagePct * 80 / 100
 	di, err := disk.GetInfo(cfs.dir)
 	if err != nil {
-		errorIf(err, "Error getting disk information on %s", cfs.dir)
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
@@ -128,7 +129,9 @@ func (cfs *cacheFSObjects) diskUsageLow() bool {
 func (cfs *cacheFSObjects) diskUsageHigh() bool {
 	di, err := disk.GetInfo(cfs.dir)
 	if err != nil {
-		errorIf(err, "Error getting disk information on %s", cfs.dir)
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
 		return true
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
@@ -140,7 +143,9 @@ func (cfs *cacheFSObjects) diskUsageHigh() bool {
 func (cfs *cacheFSObjects) diskAvailable(size int64) bool {
 	di, err := disk.GetInfo(cfs.dir)
 	if err != nil {
-		errorIf(err, "Error getting disk information on %s", cfs.dir)
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
 		return false
 	}
 	usedPercent := (di.Total - (di.Free - uint64(size))) * 100 / di.Total
@@ -150,11 +155,11 @@ func (cfs *cacheFSObjects) diskAvailable(size int64) bool {
 // purges all content marked trash from the cache.
 func (cfs *cacheFSObjects) purgeTrash() {
 	ticker := time.NewTicker(time.Minute * cacheCleanupInterval)
+	defer ticker.Stop()
+
 	for {
 		select {
 		case <-globalServiceDoneCh:
-			// Stop the timer.
-			ticker.Stop()
 			return
 		case <-ticker.C:
 			trashPath := path.Join(cfs.fsPath, minioMetaBucket, cacheTrashDir)
@@ -163,14 +168,15 @@ func (cfs *cacheFSObjects) purgeTrash() {
 				return
 			}
 			for _, entry := range entries {
-				fi, err := fsStatVolume(pathJoin(trashPath, entry))
+				ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
+				fi, err := fsStatVolume(ctx, pathJoin(trashPath, entry))
 				if err != nil {
 					continue
 				}
 				dir := path.Join(trashPath, fi.Name())
 
 				// Delete all expired cache content.
-				fsRemoveAll(dir)
+				fsRemoveAll(ctx, dir)
 			}
 		}
 	}
@@ -193,7 +199,7 @@ func (cfs *cacheFSObjects) purge() {
 			deletedCount := 0
 			buckets, err := cfs.ListBuckets(ctx)
 			if err != nil {
-				errorIf(err, "Unable to list buckets.")
+				logger.LogIf(ctx, err)
 			}
 			// Reset cache online status if drive was offline earlier.
 			if !cfs.IsOnline() {
@@ -221,7 +227,7 @@ func (cfs *cacheFSObjects) purge() {
 							continue
 						}
 						if err = cfs.DeleteObject(ctx, bucket.Name, object.Name); err != nil {
-							errorIf(err, "Unable to remove cache entry in dir %s/%s", bucket.Name, object.Name)
+							logger.LogIf(ctx, err)
 							continue
 						}
 						deletedCount++
@@ -271,7 +277,7 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data
 	}
 	_, err := cfs.PutObject(ctx, bucket, object, data, metadata)
 	// if err is due to disk being offline , mark cache drive as offline
-	if errors2.IsErr(err, baseErrs...) {
+	if IsErr(err, baseErrs...) {
 		cfs.setOnline(false)
 	}
 	return err
@@ -313,7 +319,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	var err error
 
 	// Validate if bucket name is valid and exists.
-	if _, err = fs.statBucketDir(bucket); err != nil {
+	if _, err = fs.statBucketDir(ctx, bucket); err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket)
 	}
 
@@ -325,31 +331,32 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	// and return success.
 	if isObjectDir(object, data.Size()) {
 		// Check if an object is present as one of the parent dir.
-		if fs.parentDirIsObject(bucket, path.Dir(object)) {
-			return ObjectInfo{}, toObjectErr(errors2.Trace(errFileAccessDenied), bucket, object)
+		if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
+			return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
 		}
 		if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 		var fi os.FileInfo
-		if fi, err = fsStatDir(pathJoin(fs.fsPath, bucket, object)); err != nil {
+		if fi, err = fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object)); err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 		return fsMeta.ToObjectInfo(bucket, object, fi), nil
 	}
 
-	if err = checkPutObjectArgs(bucket, object, fs, data.Size()); err != nil {
+	if err = checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil {
 		return ObjectInfo{}, err
 	}
 
 	// Check if an object is present as one of the parent dir.
-	if fs.parentDirIsObject(bucket, path.Dir(object)) {
-		return ObjectInfo{}, toObjectErr(errors2.Trace(errFileAccessDenied), bucket, object)
+	if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
+		return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
 	}
 
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < 0 {
-		return ObjectInfo{}, errors2.Trace(errInvalidArgument)
+		logger.LogIf(ctx, errInvalidArgument)
+		return ObjectInfo{}, errInvalidArgument
 	}
 
 	var wlk *lock.LockedFile
@@ -359,7 +366,8 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 
 		wlk, err = fs.rwPool.Create(fsMetaPath)
 		if err != nil {
-			return ObjectInfo{}, toObjectErr(errors2.Trace(err), bucket, object)
+			logger.LogIf(ctx, err)
+			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 		// This close will allow for locks to be synchronized on `fs.json`.
 		defer wlk.Close()
@@ -367,7 +375,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 			// Remove meta file when PutObject encounters any error
 			if retErr != nil {
 				tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)
-				fsRemoveMeta(bucketMetaDir, fsMetaPath, tmpDir)
+				fsRemoveMeta(ctx, bucketMetaDir, fsMetaPath, tmpDir)
 			}
 		}()
 	}
@@ -385,10 +393,9 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 
 	buf := make([]byte, int(bufSize))
 	fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj)
-	bytesWritten, err := fsCreateFile(fsTmpObjPath, data, buf, data.Size())
+	bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, buf, data.Size())
 	if err != nil {
-		fsRemoveFile(fsTmpObjPath)
-		errorIf(err, "Failed to create object %s/%s", bucket, object)
+		fsRemoveFile(ctx, fsTmpObjPath)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 	if fsMeta.Meta["etag"] == "" {
@@ -397,18 +404,18 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.
 	if bytesWritten < data.Size() {
-		fsRemoveFile(fsTmpObjPath)
-		return ObjectInfo{}, errors2.Trace(IncompleteBody{})
+		fsRemoveFile(ctx, fsTmpObjPath)
+		return ObjectInfo{}, IncompleteBody{}
 	}
 
 	// Delete the temporary object in the case of a
 	// failure. If PutObject succeeds, then there would be
 	// nothing to delete.
-	defer fsRemoveFile(fsTmpObjPath)
+	defer fsRemoveFile(ctx, fsTmpObjPath)
 
 	// Entire object was written to the temp location, now it's safe to rename it to the actual location.
 	fsNSObjPath := pathJoin(fs.fsPath, bucket, object)
-	if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil {
+	if err = fsRenameFile(ctx, fsTmpObjPath, fsNSObjPath); err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
@@ -420,7 +427,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	}
 
 	// Stat the file to fetch timestamp, size.
-	fi, err := fsStatFile(pathJoin(fs.fsPath, bucket, object))
+	fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
 	if err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
@@ -447,11 +454,11 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec
 		}
 	}
 	fs := cfs.FSObjects
-	if err := checkNewMultipartArgs(bucket, object, fs); err != nil {
+	if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil {
 		return "", toObjectErr(err, bucket)
 	}
 
-	if _, err := fs.statBucketDir(bucket); err != nil {
+	if _, err := fs.statBucketDir(ctx, bucket); err != nil {
 		return "", toObjectErr(err, bucket)
 	}
 
@@ -459,7 +466,8 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec
 
 	err := mkdirAll(uploadIDDir, 0755)
 	if err != nil {
-		return "", errors2.Trace(err)
+		logger.LogIf(ctx, err)
+		return "", err
 	}
 
 	// Initialize fs.json values.
@@ -468,11 +476,13 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec
 
 	fsMetaBytes, err := json.Marshal(fsMeta)
 	if err != nil {
-		return "", errors2.Trace(err)
+		logger.LogIf(ctx, err)
+		return "", err
 	}
 
 	if err = ioutil.WriteFile(pathJoin(uploadIDDir, fs.metaJSONFile), fsMetaBytes, 0644); err != nil {
-		return "", errors2.Trace(err)
+		logger.LogIf(ctx, err)
+		return "", err
 	}
 	return uploadID, nil
 }
@@ -485,7 +495,7 @@ func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string)
 		return err
 	}
 	defer bucketLock.Unlock()
-	bucketDir, err := fs.getBucketDir(bucket)
+	bucketDir, err := fs.getBucketDir(ctx, bucket)
 	if err != nil {
 		return toObjectErr(err, bucket)
 	}
@@ -493,12 +503,13 @@ func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string)
 	expiredDir := path.Join(trashPath, bucket)
 	// Attempt to move regular bucket to expired directory.
 	if err = fsRenameDir(bucketDir, expiredDir); err != nil {
+		logger.LogIf(ctx, err)
 		return toObjectErr(err, bucket)
 	}
 	// Cleanup all the bucket metadata.
 	ominioMetadataBucketDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket)
 	nminioMetadataBucketDir := pathJoin(trashPath, MustGetUUID())
-	_ = fsRenameDir(ominioMetadataBucketDir, nminioMetadataBucketDir)
+	logger.LogIf(ctx, fsRenameDir(ominioMetadataBucketDir, nminioMetadataBucketDir))
 	return nil
 }
 
@@ -506,22 +517,22 @@ func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string)
 // paths for windows automatically.
 func fsRenameDir(dirPath, newPath string) (err error) {
 	if dirPath == "" || newPath == "" {
-		return errors2.Trace(errInvalidArgument)
+		return errInvalidArgument
 	}
 
 	if err = checkPathLength(dirPath); err != nil {
-		return errors2.Trace(err)
+		return err
 	}
 	if err = checkPathLength(newPath); err != nil {
-		return errors2.Trace(err)
+		return err
 	}
 	if err = os.Rename(dirPath, newPath); err != nil {
 		if os.IsNotExist(err) {
-			return errors2.Trace(errVolumeNotFound)
+			return errVolumeNotFound
 		} else if isSysErrNotEmpty(err) {
-			return errors2.Trace(errVolumeNotEmpty)
+			return errVolumeNotEmpty
 		}
-		return errors2.Trace(err)
+		return err
 	}
 	return nil
 }

cmd/disk-cache.go

@@ -31,7 +31,7 @@ import (
 
 	"github.com/djherbis/atime"
 
-	errors2 "github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/wildcard"
 
 	"github.com/minio/minio/pkg/hash"
@@ -44,7 +44,6 @@ const (
 	// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
 	cacheSizeMultiplier  = 100
 	cacheTrashDir        = "trash"
-	cacheMaxDiskUsagePct = 80 // in %
 	cacheCleanupInterval = 10 // in minutes
 )
 
@@ -77,6 +76,13 @@ type cacheObjects struct {
 	DeleteBucketFn            func(ctx context.Context, bucket string) error
 }
 
+// CacheStorageInfo - represents total, free capacity of
+// underlying cache storage.
+type CacheStorageInfo struct {
+	Total uint64 // Total cache disk space.
+	Free  uint64 // Free cache available space.
+}
+
 // CacheObjectLayer implements primitives for cache object API layer.
 type CacheObjectLayer interface {
 	// Bucket operations.
@@ -98,13 +104,13 @@ type CacheObjectLayer interface {
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
 
 	// Storage operations.
-	StorageInfo(ctx context.Context) StorageInfo
+	StorageInfo(ctx context.Context) CacheStorageInfo
 }
 
 // backendDownError returns true if err is due to backend failure or faulty disk if in server mode
 func backendDownError(err error) bool {
-	_, backendDown := errors2.Cause(err).(BackendDown)
-	return backendDown || errors2.IsErr(err, baseErrs...)
+	_, backendDown := err.(BackendDown)
+	return backendDown || IsErr(err, baseErrs...)
 }
 
 // get cache disk where object is currently cached for a GET operation. If object does not exist at that location,
@@ -191,7 +197,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 	objInfo, err := GetObjectInfoFn(ctx, bucket, object)
 	backendDown := backendDownError(err)
 	if err != nil && !backendDown {
-		if _, ok := errors2.Cause(err).(ObjectNotFound); ok {
+		if _, ok := err.(ObjectNotFound); ok {
 			// Delete the cached entry if backend object was deleted.
 			dcache.Delete(ctx, bucket, object)
 		}
@@ -232,7 +238,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 			pipeWriter.CloseWithError(err)
 			return
 		}
-		pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
+		pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
 	}()
 	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo))
 	if err != nil {
@@ -255,7 +261,7 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string)
 	}
 	objInfo, err := getObjectInfoFn(ctx, bucket, object)
 	if err != nil {
-		if _, ok := errors2.Cause(err).(ObjectNotFound); ok {
+		if _, ok := err.(ObjectNotFound); ok {
 			// Delete the cached entry if backend object was deleted.
 			dcache.Delete(ctx, bucket, object)
 			return ObjectInfo{}, err
@@ -289,12 +295,20 @@ func listDirCacheFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks [
 	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string, err error) {
 		var entries []string
 		for _, disk := range disks {
+			// ignore disk-caches that might be missing/offline
+			if disk == nil {
+				continue
+			}
 			fs := disk.FSObjects
 			entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
+
+			// For any reason disk was deleted or goes offline, continue
+			// and list from other disks if possible.
 			if err != nil {
-				// For any reason disk was deleted or goes offline, continue
-				// and list from other disks if possible.
-				continue
+				if IsErrIgnored(err, treeWalkIgnoredErrs...) {
+					continue
+				}
+				return nil, err
 			}
 
 			// Filter entries that have the prefix prefixEntry.
@@ -345,12 +359,20 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			if err != nil {
 				return false
 			}
-			_, err = fs.getObjectInfo(bucket, object)
+			_, err = fs.getObjectInfo(ctx, bucket, object)
 			return err == nil
 		}
 
+		isLeafDir := func(bucket, object string) bool {
+			fs, err := c.cache.getCacheFS(ctx, bucket, object)
+			if err != nil {
+				return false
+			}
+			return fs.isObjectDir(bucket, object)
+		}
+
 		listDir := listDirCacheFactory(isLeaf, cacheTreeWalkIgnoredErrs, c.cache.cfs)
-		walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh)
+		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 
 	for i := 0; i < maxKeys; {
@@ -377,16 +399,16 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			var err error
 			fs, err := c.cache.getCacheFS(ctx, bucket, entry)
 			if err != nil {
-				// Ignore errFileNotFound
-				if errors2.Cause(err) == errFileNotFound {
+				// Ignore errDiskNotFound
+				if err == errDiskNotFound {
 					continue
 				}
 				return result, toObjectErr(err, bucket, prefix)
 			}
-			objInfo, err = fs.getObjectInfo(bucket, entry)
+			objInfo, err = fs.getObjectInfo(ctx, bucket, entry)
 			if err != nil {
-				// Ignore errFileNotFound
-				if errors2.Cause(err) == errFileNotFound {
+				// Ignore ObjectNotFound error
+				if _, ok := err.(ObjectNotFound); ok {
 					continue
 				}
 				return result, toObjectErr(err, bucket, prefix)
@@ -409,7 +431,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 	result = ListObjectsInfo{IsTruncated: !eof}
 	for _, objInfo := range objInfos {
 		result.NextMarker = objInfo.Name
-		if objInfo.IsDir {
+		if objInfo.IsDir && delimiter == slashSeparator {
 			result.Prefixes = append(result.Prefixes, objInfo.Name)
 			continue
 		}
@@ -469,6 +491,10 @@ func (c cacheObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continu
 func (c cacheObjects) listBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
 	m := make(map[string]string)
 	for _, cache := range c.cache.cfs {
+		// ignore disk-caches that might be missing/offline
+		if cache == nil {
+			continue
+		}
 		entries, err := cache.ListBuckets(ctx)
 
 		if err != nil {
@@ -507,6 +533,10 @@ func (c cacheObjects) GetBucketInfo(ctx context.Context, bucket string) (bucketI
 	bucketInfo, err = getBucketInfoFn(ctx, bucket)
 	if backendDownError(err) {
 		for _, cache := range c.cache.cfs {
+			// ignore disk-caches that might be missing/offline
+			if cache == nil {
+				continue
+			}
 			if bucketInfo, err = cache.GetBucketInfo(ctx, bucket); err == nil {
 				return
 			}
@@ -747,23 +777,22 @@ func (c cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, objec
 }
 
 // StorageInfo - returns underlying storage statistics.
-func (c cacheObjects) StorageInfo(ctx context.Context) (storageInfo StorageInfo) {
+func (c cacheObjects) StorageInfo(ctx context.Context) (cInfo CacheStorageInfo) {
 	var total, free uint64
 	for _, cfs := range c.cache.cfs {
 		if cfs == nil {
 			continue
 		}
-		info, err := getDiskInfo((cfs.fsPath))
-		errorIf(err, "Unable to get disk info %#v", cfs.fsPath)
+		info, err := getDiskInfo(cfs.fsPath)
+		logger.GetReqInfo(ctx).AppendTags("cachePath", cfs.fsPath)
+		logger.LogIf(ctx, err)
 		total += info.Total
 		free += info.Free
 	}
-	storageInfo = StorageInfo{
+	return CacheStorageInfo{
 		Total: total,
 		Free:  free,
 	}
-	storageInfo.Backend.Type = FS
-	return storageInfo
 }
 
 // DeleteBucket - marks bucket to be deleted from cache if bucket is deleted from backend.
@@ -771,6 +800,10 @@ func (c cacheObjects) DeleteBucket(ctx context.Context, bucket string) (err erro
 	deleteBucketFn := c.DeleteBucketFn
 	var toDel []*cacheFSObjects
 	for _, cfs := range c.cache.cfs {
+		// ignore disk-caches that might be missing/offline
+		if cfs == nil {
+			continue
+		}
 		if _, cerr := cfs.GetBucketInfo(ctx, bucket); cerr == nil {
 			toDel = append(toDel, cfs)
 		}
@@ -791,7 +824,8 @@ func (c cacheObjects) DeleteBucket(ctx context.Context, bucket string) (err erro
 // or the global env overrides.
 func newCache(config CacheConfig) (*diskCache, error) {
 	var cfsObjects []*cacheFSObjects
-	formats, err := loadAndValidateCacheFormat(config.Drives)
+	ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
+	formats, err := loadAndValidateCacheFormat(ctx, config.Drives)
 	if err != nil {
 		return nil, err
 	}
@@ -804,7 +838,7 @@ func newCache(config CacheConfig) (*diskCache, error) {
 		if err := checkAtimeSupport(dir); err != nil {
 			return nil, errors.New("Atime support required for disk caching")
 		}
-		cache, err := newCacheFSObjects(dir, config.Expiry, cacheMaxDiskUsagePct)
+		cache, err := newCacheFSObjects(dir, config.Expiry, config.MaxUse)
 		if err != nil {
 			return nil, err
 		}

cmd/disk-cache_test.go

@@ -28,10 +28,10 @@ import (
 )
 
 // Initialize cache FS objects.
-func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
+func initCacheFSObjects(disk string, cacheMaxUse int, t *testing.T) (*cacheFSObjects, error) {
 	newTestConfig(globalMinioDefaultRegion)
 	var err error
-	obj, err := newCacheFSObjects(disk, globalCacheExpiry, 100)
+	obj, err := newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -39,10 +39,10 @@ func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
 }
 
 // inits diskCache struct for nDisks
-func initDiskCaches(drives []string, t *testing.T) (*diskCache, error) {
+func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
 	var cfs []*cacheFSObjects
 	for _, d := range drives {
-		obj, err := initCacheFSObjects(d, t)
+		obj, err := initCacheFSObjects(d, cacheMaxUse, t)
 		if err != nil {
 			return nil, err
 		}
@@ -59,7 +59,46 @@ func TestGetCacheFS(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, t)
+		d, err := initDiskCaches(fsDirs, 100, t)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bucketName := "testbucket"
+		objectName := "testobject"
+		ctx := context.Background()
+		// find cache drive where object would be hashed
+		index := d.hashIndex(bucketName, objectName)
+		// turn off drive by setting online status to false
+		d.cfs[index].online = false
+		cfs, err := d.getCacheFS(ctx, bucketName, objectName)
+		if n == 1 && err == errDiskNotFound {
+			continue
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		i := -1
+		for j, f := range d.cfs {
+			if f == cfs {
+				i = j
+				break
+			}
+		}
+		if i != (index+1)%n {
+			t.Fatalf("expected next cache location to be picked")
+		}
+	}
+}
+
+// test whether a drive being offline causes
+// getCacheFS to fetch next online drive
+func TestGetCacheFSMaxUse(t *testing.T) {
+	for n := 1; n < 10; n++ {
+		fsDirs, err := getRandomDisks(n)
+		if err != nil {
+			t.Fatal(err)
+		}
+		d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -141,7 +180,7 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, t)
+	d, err := initDiskCaches(fsDirs, 100, t)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -209,6 +248,87 @@ func TestDiskCache(t *testing.T) {
 	}
 }
 
+// Test diskCache with upper bound on max cache use.
+func TestDiskCacheMaxUse(t *testing.T) {
+	fsDirs, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cache := d.cfs[0]
+	ctx := context.Background()
+	bucketName := "testbucket"
+	objectName := "testobject"
+	content := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	etag := "061208c10af71a30c6dcd6cf5d89f0fe"
+	contentType := "application/zip"
+	size := len(content)
+
+	httpMeta := make(map[string]string)
+	httpMeta["etag"] = etag
+	httpMeta["content-type"] = contentType
+
+	objInfo := ObjectInfo{}
+	objInfo.Bucket = bucketName
+	objInfo.Name = objectName
+	objInfo.Size = int64(size)
+	objInfo.ContentType = contentType
+	objInfo.ETag = etag
+	objInfo.UserDefined = httpMeta
+
+	byteReader := bytes.NewReader([]byte(content))
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !cache.diskAvailable(int64(size)) {
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		if err != errDiskFull {
+			t.Fatal("Cache max-use limit violated.")
+		}
+	} else {
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		if err != nil {
+			t.Fatal(err)
+		}
+		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !cache.Exists(ctx, bucketName, objectName) {
+			t.Fatal("Expected object to exist on cache")
+		}
+		if cachedObjInfo.ETag != objInfo.ETag {
+			t.Fatal("Expected ETag to match")
+		}
+		if cachedObjInfo.Size != objInfo.Size {
+			t.Fatal("Size mismatch")
+		}
+		if cachedObjInfo.ContentType != objInfo.ContentType {
+			t.Fatal("Cached content-type does not match")
+		}
+		writer := bytes.NewBuffer(nil)
+		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
+			t.Errorf("wrong cached file content")
+		}
+		err = cache.Delete(ctx, bucketName, objectName)
+		if err != nil {
+			t.Errorf("object missing from cache")
+		}
+		online := cache.IsOnline()
+		if !online {
+			t.Errorf("expected cache drive to be online")
+		}
+	}
+}
+
 func TestIsCacheExcludeDirective(t *testing.T) {
 	testCases := []struct {
 		cacheControlOpt string

cmd/disk-usage.go

@@ -0,0 +1,54 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+)
+
+// getDiskUsage walks the file tree rooted at root, calling usageFn
+// for each file or directory in the tree, including root.
+func getDiskUsage(ctx context.Context, root string, usageFn usageFunc) error {
+	return walk(ctx, root+slashSeparator, usageFn)
+}
+
+type usageFunc func(ctx context.Context, entry string) error
+
+// walk recursively descends path, calling walkFn.
+func walk(ctx context.Context, path string, usageFn usageFunc) error {
+	if err := usageFn(ctx, path); err != nil {
+		return err
+	}
+
+	if !hasSuffix(path, slashSeparator) {
+		return nil
+	}
+
+	entries, err := readDir(path)
+	if err != nil {
+		return usageFn(ctx, path)
+	}
+
+	for _, entry := range entries {
+		fname := pathJoin(path, entry)
+		if err = walk(ctx, fname, usageFn); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/dummy-object-layer_test.go

@@ -0,0 +1,165 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/policy"
+)
+
+type DummyObjectLayer struct{}
+
+func (api *DummyObjectLayer) Shutdown(context.Context) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) StorageInfo(context.Context) (si StorageInfo) {
+	return
+}
+
+func (api *DummyObjectLayer) MakeBucketWithLocation(ctx context.Context, bucket string, location string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) DeleteBucket(ctx context.Context, bucket string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) DeleteObject(ctx context.Context, bucket, object string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo) (info PartInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) HealFormat(ctx context.Context, dryRun bool) (item madmin.HealResultItem, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) HealBucket(ctx context.Context, bucket string, dryRun bool) (items []madmin.HealResultItem, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) HealObject(ctx context.Context, bucket, object string, dryRun bool) (item madmin.HealResultItem, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (info ListObjectsInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) (info []VolumeLockInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ClearLocks(context.Context, []VolumeLockInfo) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) SetBucketPolicy(context.Context, string, *policy.Policy) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) GetBucketPolicy(context.Context, string) (bucketPolicy *policy.Policy, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) RefreshBucketPolicy(context.Context, string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) DeleteBucketPolicy(context.Context, string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) IsNotificationSupported() (b bool) {
+	return
+}
+
+func (api *DummyObjectLayer) IsEncryptionSupported() (b bool) {
+	return
+}

cmd/encryption-v1.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/hmac"
 	"crypto/md5"
 	"crypto/rand"
@@ -27,7 +28,9 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"strconv"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"
 	sha256 "github.com/minio/sha256-simd"
 	"github.com/minio/sio"
@@ -35,14 +38,15 @@ import (
 
 var (
 	// AWS errors for invalid SSE-C requests.
-	errInsecureSSERequest  = errors.New("Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection")
-	errEncryptedObject     = errors.New("The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object")
-	errInvalidSSEAlgorithm = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm")
-	errMissingSSEKey       = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key")
-	errInvalidSSEKey       = errors.New("The secret key was invalid for the specified algorithm")
-	errMissingSSEKeyMD5    = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key")
-	errSSEKeyMD5Mismatch   = errors.New("The calculated MD5 hash of the key did not match the hash that was provided")
-	errSSEKeyMismatch      = errors.New("The client provided key does not match the key provided when the object was encrypted") // this msg is not shown to the client
+	errInsecureSSERequest   = errors.New("SSE-C requests require TLS connections")
+	errEncryptedObject      = errors.New("The object was stored using a form of SSE")
+	errInvalidSSEAlgorithm  = errors.New("The SSE-C algorithm is not valid")
+	errMissingSSEKey        = errors.New("The SSE-C request is missing the customer key")
+	errInvalidSSEKey        = errors.New("The SSE-C key is invalid")
+	errMissingSSEKeyMD5     = errors.New("The SSE-C request is missing the customer key MD5")
+	errSSEKeyMD5Mismatch    = errors.New("The key MD5 does not match the SSE-C key")
+	errSSEKeyMismatch       = errors.New("The SSE-C key is not correct")                  // access denied
+	errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
 
 	// Additional Minio errors for SSE-C requests.
 	errObjectTampered = errors.New("The requested object was modified and may be compromised")
@@ -247,9 +251,6 @@ func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
 
 // This function rotates old to new key.
 func rotateKey(oldKey []byte, newKey []byte, metadata map[string]string) error {
-	if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
-		return nil
-	}
 	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
 
 	if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
@@ -273,10 +274,14 @@ func rotateKey(oldKey []byte, newKey []byte, metadata map[string]string) error {
 	n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
 		Key: keyEncryptionKey,
 	})
-	if n != 32 || err != nil {
-		// Either the provided key does not match or the object was tampered.
-		// To provide strict AWS S3 compatibility we return: access denied.
-		return errSSEKeyMismatch
+	if n != 32 || err != nil { // Either the provided key does not match or the object was tampered.
+		if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
+			return errInvalidSSEParameters // AWS returns special error for equal but invalid keys.
+		}
+		return errSSEKeyMismatch // To provide strict AWS S3 compatibility we return: access denied.
+	}
+	if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
+		return nil // we don't need to rotate keys if newKey == oldKey
 	}
 
 	nonce := make([]byte, 32) // generate random values for key derivation
@@ -737,10 +742,9 @@ func (li *ListPartsInfo) IsEncrypted() bool {
 // DecryptedSize returns the size of the object after decryption in bytes.
 // It returns an error if the object is not encrypted or marked as encrypted
 // but has an invalid size.
-// DecryptedSize panics if the referred object is not encrypted.
 func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if !o.IsEncrypted() {
-		panic("cannot compute decrypted size of an object which is not encrypted")
+		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
 	size, err := sio.DecryptedSize(uint64(o.Size))
 	if err != nil {
@@ -755,7 +759,11 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 func (o *ObjectInfo) EncryptedSize() int64 {
 	size, err := sio.EncryptedSize(uint64(o.Size))
 	if err != nil {
-		panic(err) // Since AWS S3 allows parts to be 5GB at most this cannot happen - sio max. size is 256 TB
+		// This cannot happen since AWS S3 allows parts to be 5GB at most
+		// sio max. size is 256 TB
+		reqInfo := (&logger.ReqInfo{}).AppendTags("size", strconv.FormatUint(size, 10))
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.CriticalIf(ctx, err)
 	}
 	return int64(size)
 }

cmd/endpoint-ellipses.go

@@ -70,11 +70,10 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 	}
 
 	setIndexes = make([][]uint64, len(totalSizes))
-	for i, totalSize := range totalSizes {
+	for _, totalSize := range totalSizes {
 		// Check if totalSize has minimum range upto setSize
 		if totalSize < setSizes[0] {
-			return nil, fmt.Errorf("Invalid inputs (%s). Ellipses range or number of args %d should be atleast divisible by least possible set size %d",
-				args[i], totalSize, setSizes[0])
+			return nil, uiErrInvalidNumberOfErasureEndpoints(nil)
 		}
 	}
 
@@ -103,8 +102,7 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		return nil, fmt.Errorf("Invalid inputs (%s). Ellipses range or number of args %d should be atleast divisible by least possible set size %d",
-			args, setSize, setSizes[0])
+		return nil, uiErrInvalidNumberOfErasureEndpoints(nil)
 	}
 
 	for i := range totalSizes {
@@ -167,14 +165,14 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) {
 	for i, arg := range args {
 		patterns, perr := ellipses.FindEllipsesPatterns(arg)
 		if perr != nil {
-			return endpointSet{}, perr
+			return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(perr.Error())
 		}
 		argPatterns[i] = patterns
 	}
 
 	ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns))
 	if err != nil {
-		return endpointSet{}, err
+		return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}
 
 	ep.argPatterns = argPatterns
@@ -223,7 +221,7 @@ func getAllSets(args ...string) ([][]string, error) {
 	for _, sargs := range setArgs {
 		for _, arg := range sargs {
 			if uniqueArgs.Contains(arg) {
-				return nil, fmt.Errorf("Input args (%s) has duplicate ellipses", args)
+				return nil, uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args))
 			}
 			uniqueArgs.Add(arg)
 		}

cmd/endpoint.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"os"
 	"path"
 	"path/filepath"
 	"runtime"
@@ -27,6 +28,7 @@ import (
 	"strings"
 
 	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/mountinfo"
 )
 
@@ -251,14 +253,14 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			return serverAddr, endpoints, setupType, err
 		}
 		if endpoint.Type() != PathEndpointType {
-			return serverAddr, endpoints, setupType, fmt.Errorf("use path style endpoint for FS setup")
+			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
 		}
 		endpoints = append(endpoints, endpoint)
 		setupType = FSSetupType
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(endpoints); err != nil {
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg(err.Error())
 		}
 		return serverAddr, endpoints, setupType, nil
 	}
@@ -269,12 +271,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		var eps EndpointList
 		eps, err = NewEndpointList(iargs...)
 		if err != nil {
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(eps); err != nil {
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
 		for _, ep := range eps {
@@ -315,7 +317,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 
 	// No local endpoint found.
 	if localEndpointCount == 0 {
-		return serverAddr, endpoints, setupType, fmt.Errorf("no endpoint found for this host")
+		return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg("no endpoint pointing to the local machine is found")
 	}
 
 	// Check whether same path is not used in endpoints of a host on different port.
@@ -330,8 +332,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			hostIPSet, _ := getHostIP4(host)
 			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
 				if !IPSet.Intersection(hostIPSet).IsEmpty() {
-					err = fmt.Errorf("path '%s' can not be served by different port on same address", endpoint.Path)
-					return serverAddr, endpoints, setupType, err
+					return serverAddr, endpoints, setupType,
+						uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
 				}
 				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
 			} else {
@@ -348,8 +350,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 				continue
 			}
 			if localPathSet.Contains(endpoint.Path) {
-				err = fmt.Errorf("path '%s' cannot be served by different address on same server", endpoint.Path)
-				return serverAddr, endpoints, setupType, err
+				return serverAddr, endpoints, setupType,
+					uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
 			}
 			localPathSet.Add(endpoint.Path)
 		}
@@ -359,12 +361,11 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 	{
 		if !localPortSet.Contains(serverAddrPort) {
 			if len(localPortSet) > 1 {
-				err = fmt.Errorf("port number in server address must match with one of the port in local endpoints")
-			} else {
-				err = fmt.Errorf("server address and local endpoint have different ports")
+				return serverAddr, endpoints, setupType,
+					uiErrInvalidErasureEndpoints(nil).Msg("port number in server address must match with one of the port in local endpoints")
 			}
-
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType,
+				uiErrInvalidErasureEndpoints(nil).Msg("server address and local endpoint have different ports")
 		}
 	}
 
@@ -373,7 +374,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		// If all endpoints have same port number, then this is XL setup using URL style endpoints.
 		if len(localPortSet) == 1 {
 			if len(localServerAddrSet) > 1 {
-				// TODO: Eventhough all endpoints are local, the local host is referred by different IP/name.
+				// TODO: Even though all endpoints are local, the local host is referred by different IP/name.
 				// eg '172.0.0.1', 'localhost' and 'mylocalhostname' point to same local host.
 				//
 				// In this case, we bind to 0.0.0.0 ie to all interfaces.
@@ -387,7 +388,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			return serverAddr, endpoints, setupType, nil
 		}
 
-		// Eventhough all endpoints are local, but those endpoints use different ports.
+		// Even though all endpoints are local, but those endpoints use different ports.
 		// This means it is DistXL setup.
 	} else {
 		// This is DistXL setup.
@@ -399,7 +400,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			}
 
 			ipList, err := getHostIP4(host)
-			fatalIf(err, "unexpected error when resolving host '%s'", host)
+			logger.FatalIf(err, "unexpected error when resolving host '%s'", host)
 
 			// Filter ipList by IPs those start with '127.'.
 			loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool {
@@ -442,6 +443,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		return serverAddr, endpoints, setupType, err
 	}
 
+	updateDomainIPs(uniqueArgs)
+
 	setupType = DistXLSetupType
 	return serverAddr, endpoints, setupType, nil
 }
@@ -493,3 +496,22 @@ func GetRemotePeers(endpoints EndpointList) []string {
 
 	return peerSet.ToSlice()
 }
+
+// In federated and distributed setup, update IP addresses of the hosts passed in command line
+// if MINIO_PUBLIC_IPS are not set manually
+func updateDomainIPs(endPoints set.StringSet) {
+	_, dok := os.LookupEnv("MINIO_DOMAIN")
+	_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if dok && eok && !iok {
+		globalDomainIPs = set.NewStringSet()
+		for e := range endPoints {
+			host, _, _ := net.SplitHostPort(e)
+			ipList, _ := getHostIP4(host)
+			remoteIPList := ipList.FuncMatch(func(ip string, matchString string) bool {
+				return !strings.HasPrefix(ip, "127.")
+			}, "")
+			globalDomainIPs.Add(remoteIPList.ToSlice()[0])
+		}
+	}
+}

cmd/endpoint_test.go

@@ -225,8 +225,8 @@ func TestCreateEndpoints(t *testing.T) {
 		{"localhost:10000", [][]string{{"./d1"}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
 		{"localhost:10000", [][]string{{`\d1`}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `\d1`}, IsLocal: true}}, FSSetupType, nil},
 		{"localhost:10000", [][]string{{`.\d1`}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `.\d1`}, IsLocal: true}}, FSSetupType, nil},
-		{":8080", [][]string{{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
-		{":8080", [][]string{{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
+		{":8080", [][]string{{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint pointing to the local machine is found")},
+		{":8080", [][]string{{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint pointing to the local machine is found")},
 		{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
 		{"localhost:9000", [][]string{{"https://127.0.0.1:8000/d1", "https://localhost:9001/d2", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("port number in server address must match with one of the port in local endpoints")},
 		{"localhost:10000", [][]string{{"https://127.0.0.1:8000/d1", "https://localhost:8000/d2", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("server address and local endpoint have different ports")},

cmd/erasure-createfile.go

@@ -17,18 +17,20 @@
 package cmd
 
 import (
+	"context"
 	"hash"
 	"io"
 
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // CreateFile creates a new bitrot encoded file spread over all available disks. CreateFile will create
 // the file at the given volume and path. It will read from src until an io.EOF occurs. The given algorithm will
 // be used to protect the erasure encoded file.
-func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) {
+func (s *ErasureStorage) CreateFile(ctx context.Context, src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) {
 	if !algorithm.Available() {
-		return f, errors.Trace(errBitrotHashAlgoInvalid)
+		logger.LogIf(ctx, errBitrotHashAlgoInvalid)
+		return f, errBitrotHashAlgoInvalid
 	}
 	f.Checksums = make([][]byte, len(s.disks))
 	hashers := make([]hash.Hash, len(s.disks))
@@ -50,21 +52,22 @@ func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer [
 			}
 			blocks = make([][]byte, len(s.disks)) // write empty block
 		} else if err == nil || (n > 0 && err == io.ErrUnexpectedEOF) {
-			blocks, err = s.ErasureEncode(buffer[:n])
+			blocks, err = s.ErasureEncode(ctx, buffer[:n])
 			if err != nil {
 				return f, err
 			}
 		} else {
-			return f, errors.Trace(err)
+			logger.LogIf(ctx, err)
+			return f, err
 		}
 
 		for i := range errChans { // span workers
-			go erasureAppendFile(s.disks[i], volume, path, hashers[i], blocks[i], errChans[i])
+			go erasureAppendFile(ctx, s.disks[i], volume, path, hashers[i], blocks[i], errChans[i])
 		}
 		for i := range errChans { // wait until all workers are finished
 			errs[i] = <-errChans[i]
 		}
-		if err = reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum); err != nil {
+		if err = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum); err != nil {
 			return f, err
 		}
 		s.disks = evalDisks(s.disks, errs)
@@ -83,9 +86,10 @@ func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer [
 
 // erasureAppendFile appends the content of buf to the file on the given disk and updates computes
 // the hash of the written data. It sends the write error (or nil) over the error channel.
-func erasureAppendFile(disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) {
+func erasureAppendFile(ctx context.Context, disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) {
 	if disk == OfflineDisk {
-		errChan <- errors.Trace(errDiskNotFound)
+		logger.LogIf(ctx, errDiskNotFound)
+		errChan <- errDiskNotFound
 		return
 	}
 	err := disk.AppendFile(volume, path, buf)

cmd/erasure-createfile_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"io"
 	"testing"
@@ -70,7 +71,7 @@ func TestErasureCreateFile(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		storage, err := NewErasureStorage(context.Background(), setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -82,7 +83,7 @@ func TestErasureCreateFile(t *testing.T) {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
 		}
-		file, err := storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object", buffer, test.algorithm, test.dataBlocks+1)
+		file, err := storage.CreateFile(context.Background(), bytes.NewReader(data[test.offset:]), "testbucket", "object", buffer, test.algorithm, test.dataBlocks+1)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
 		}
@@ -100,7 +101,7 @@ func TestErasureCreateFile(t *testing.T) {
 			if test.offDisks > 0 {
 				storage.disks[0] = OfflineDisk
 			}
-			file, err = storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object2", buffer, test.algorithm, test.dataBlocks+1)
+			file, err = storage.CreateFile(context.Background(), bytes.NewReader(data[test.offset:]), "testbucket", "object2", buffer, test.algorithm, test.dataBlocks+1)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
 			}
@@ -125,7 +126,7 @@ func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b
 		b.Fatalf("failed to create test setup: %v", err)
 	}
 	defer setup.Remove()
-	storage, err := NewErasureStorage(setup.disks, data, parity, blockSizeV1)
+	storage, err := NewErasureStorage(context.Background(), setup.disks, data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -143,7 +144,7 @@ func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b
 	b.SetBytes(size)
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		_, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
+		_, err := storage.CreateFile(context.Background(), bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
 		if err != nil {
 			panic(err)
 		}

cmd/erasure-healfile.go

@@ -17,11 +17,12 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"hash"
 	"strings"
 
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // HealFile tries to reconstruct an erasure-coded file spread over all
@@ -45,12 +46,13 @@ import (
 //
 // It returns bitrot checksums for the non-nil staleDisks on which
 // healing succeeded.
-func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, blocksize int64,
+func (s ErasureStorage) HealFile(ctx context.Context, staleDisks []StorageAPI, volume, path string, blocksize int64,
 	dstVol, dstPath string, size int64, alg BitrotAlgorithm, checksums [][]byte) (
 	f ErasureFileInfo, err error) {
 
 	if !alg.Available() {
-		return f, errors.Trace(errBitrotHashAlgoInvalid)
+		logger.LogIf(ctx, errBitrotHashAlgoInvalid)
+		return f, errBitrotHashAlgoInvalid
 	}
 
 	// Initialization
@@ -84,7 +86,7 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
 	}
 	readLen += lastChunkSize
 	var buffers [][]byte
-	buffers, _, err = s.readConcurrent(volume, path, 0, readLen, verifiers)
+	buffers, _, err = s.readConcurrent(ctx, volume, path, 0, readLen, verifiers)
 	if err != nil {
 		return f, err
 	}
@@ -131,7 +133,7 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
 		}
 		buffOffset += csize
 
-		if err = s.ErasureDecodeDataAndParityBlocks(blocks); err != nil {
+		if err = s.ErasureDecodeDataAndParityBlocks(ctx, blocks); err != nil {
 			return f, err
 		}
 
@@ -155,7 +157,9 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
 		// If all disks had write errors we quit.
 		if !writeSucceeded {
 			// build error from all write errors
-			return f, errors.Trace(joinWriteErrors(writeErrors))
+			err := joinWriteErrors(writeErrors)
+			logger.LogIf(ctx, err)
+			return f, err
 		}
 	}
 

cmd/erasure-healfile_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"io"
 	"reflect"
@@ -74,7 +75,7 @@ func TestErasureHealFile(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
 		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
+		storage, err := NewErasureStorage(context.Background(), setup.disks, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -89,7 +90,7 @@ func TestErasureHealFile(t *testing.T) {
 			algorithm = DefaultBitrotAlgorithm
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
-		file, err := storage.CreateFile(bytes.NewReader(data), "testbucket", "testobject", buffer, algorithm, test.dataBlocks+1)
+		file, err := storage.CreateFile(context.Background(), bytes.NewReader(data), "testbucket", "testobject", buffer, algorithm, test.dataBlocks+1)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
@@ -113,7 +114,7 @@ func TestErasureHealFile(t *testing.T) {
 		}
 
 		// test case setup is complete - now call Healfile()
-		info, err := storage.HealFile(staleDisks, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums)
+		info, err := storage.HealFile(context.Background(), staleDisks, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but it failed with: %v", i, err)
 		}

cmd/erasure-readfile.go

@@ -17,9 +17,10 @@
 package cmd
 
 import (
+	"context"
 	"io"
 
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 type errIdx struct {
@@ -27,7 +28,7 @@ type errIdx struct {
 	err error
 }
 
-func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64,
+func (s ErasureStorage) readConcurrent(ctx context.Context, volume, path string, offset, length int64,
 	verifiers []*BitrotVerifier) (buffers [][]byte, needsReconstruction bool,
 	err error) {
 
@@ -39,7 +40,8 @@ func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64
 		stageBuffers[i] = make([]byte, length)
 		disk := s.disks[i]
 		if disk == OfflineDisk {
-			errChan <- errIdx{i, errors.Trace(errDiskNotFound)}
+			logger.LogIf(ctx, errDiskNotFound)
+			errChan <- errIdx{i, errDiskNotFound}
 			return
 		}
 		_, rerr := disk.ReadFile(volume, path, offset, stageBuffers[i], verifiers[i])
@@ -75,7 +77,8 @@ func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64
 	}
 	if successCount != s.dataBlocks {
 		// Not enough disks returns data.
-		err = errors.Trace(errXLReadQuorum)
+		err = errXLReadQuorum
+		logger.LogIf(ctx, err)
 	}
 	return
 }
@@ -86,18 +89,21 @@ func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64
 // integrity of the given file. ReadFile will read data from the given
 // offset up to the given length. If parts of the file are corrupted
 // ReadFile tries to reconstruct the data.
-func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset,
+func (s ErasureStorage) ReadFile(ctx context.Context, writer io.Writer, volume, path string, offset,
 	length, totalLength int64, checksums [][]byte, algorithm BitrotAlgorithm,
 	blocksize int64) (f ErasureFileInfo, err error) {
 
 	if offset < 0 || length < 0 {
-		return f, errors.Trace(errUnexpected)
+		logger.LogIf(ctx, errUnexpected)
+		return f, errUnexpected
 	}
 	if offset+length > totalLength {
-		return f, errors.Trace(errUnexpected)
+		logger.LogIf(ctx, errUnexpected)
+		return f, errUnexpected
 	}
 	if !algorithm.Available() {
-		return f, errors.Trace(errBitrotHashAlgoInvalid)
+		logger.LogIf(ctx, errBitrotHashAlgoInvalid)
+		return f, errBitrotHashAlgoInvalid
 	}
 
 	f.Checksums = make([][]byte, len(s.disks))
@@ -145,7 +151,7 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset,
 
 	var buffers [][]byte
 	var needsReconstruction bool
-	buffers, needsReconstruction, err = s.readConcurrent(volume, path,
+	buffers, needsReconstruction, err = s.readConcurrent(ctx, volume, path,
 		partDataStartIndex, partDataLength, verifiers)
 	if err != nil {
 		// Could not read enough disks.
@@ -194,7 +200,8 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset,
 
 		if needsReconstruction {
 			if err = s.ErasureDecodeDataBlocks(blocks); err != nil {
-				return f, errors.Trace(err)
+				logger.LogIf(ctx, err)
+				return f, err
 			}
 		}
 
@@ -210,7 +217,7 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset,
 				writeLength = lastBlockLength - writeStart
 			}
 		}
-		n, err := writeDataBlocks(writer, blocks, s.dataBlocks, writeStart, writeLength)
+		n, err := writeDataBlocks(ctx, writer, blocks, s.dataBlocks, writeStart, writeLength)
 		if err != nil {
 			return f, err
 		}

cmd/erasure-readfile_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	crand "crypto/rand"
 	"io"
 	"math/rand"
@@ -86,7 +87,7 @@ func TestErasureReadFile(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		storage, err := NewErasureStorage(context.Background(), setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -102,13 +103,13 @@ func TestErasureReadFile(t *testing.T) {
 			writeAlgorithm = DefaultBitrotAlgorithm
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
-		file, err := storage.CreateFile(bytes.NewReader(data[:]), "testbucket", "object", buffer, writeAlgorithm, test.dataBlocks+1)
+		file, err := storage.CreateFile(context.Background(), bytes.NewReader(data[:]), "testbucket", "object", buffer, writeAlgorithm, test.dataBlocks+1)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file: %v", i, err)
 		}
 		writer := bytes.NewBuffer(nil)
-		readInfo, err := storage.ReadFile(writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
+		readInfo, err := storage.ReadFile(context.Background(), writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
 		}
@@ -134,7 +135,7 @@ func TestErasureReadFile(t *testing.T) {
 			if test.offDisks > 0 {
 				storage.disks[0] = OfflineDisk
 			}
-			readInfo, err = storage.ReadFile(writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
+			readInfo, err = storage.ReadFile(context.Background(), writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
 			}
@@ -174,7 +175,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	}
 	defer setup.Remove()
 
-	storage, err := NewErasureStorage(setup.disks, dataBlocks, parityBlocks, blockSize)
+	storage, err := NewErasureStorage(context.Background(), setup.disks, dataBlocks, parityBlocks, blockSize)
 	if err != nil {
 		t.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -191,7 +192,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 	// Create a test file to read from.
 	buffer := make([]byte, blockSize, 2*blockSize)
-	file, err := storage.CreateFile(bytes.NewReader(data), "testbucket", "testobject", buffer, DefaultBitrotAlgorithm, dataBlocks+1)
+	file, err := storage.CreateFile(context.Background(), bytes.NewReader(data), "testbucket", "testobject", buffer, DefaultBitrotAlgorithm, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -211,7 +212,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 		expected := data[offset : offset+readLen]
 
-		_, err = storage.ReadFile(buf, "testbucket", "testobject", offset, readLen, length, file.Checksums, DefaultBitrotAlgorithm, blockSize)
+		_, err = storage.ReadFile(context.Background(), buf, "testbucket", "testobject", offset, readLen, length, file.Checksums, DefaultBitrotAlgorithm, blockSize)
 		if err != nil {
 			t.Fatal(err, offset, readLen)
 		}
@@ -231,14 +232,14 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b
 		b.Fatalf("failed to create test setup: %v", err)
 	}
 	defer setup.Remove()
-	storage, err := NewErasureStorage(setup.disks, data, parity, blockSizeV1)
+	storage, err := NewErasureStorage(context.Background(), setup.disks, data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}
 
 	content := make([]byte, size)
 	buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
-	file, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
+	file, err := storage.CreateFile(context.Background(), bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
 	if err != nil {
 		b.Fatalf("failed to create erasure test file: %v", err)
 	}
@@ -255,7 +256,7 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b
 	b.SetBytes(size)
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		if file, err = storage.ReadFile(bytes.NewBuffer(content[:0]), "testbucket", "object", 0, size, size, checksums, DefaultBitrotAlgorithm, blockSizeV1); err != nil {
+		if file, err = storage.ReadFile(context.Background(), bytes.NewBuffer(content[:0]), "testbucket", "object", 0, size, size, checksums, DefaultBitrotAlgorithm, blockSizeV1); err != nil {
 			panic(err)
 		}
 	}

cmd/erasure-utils.go

@@ -18,10 +18,11 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"io"
 
 	"github.com/klauspost/reedsolomon"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // getDataBlockLen - get length of data blocks from encoded blocks.
@@ -36,20 +37,23 @@ func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
 
 // Writes all the data blocks from encoded blocks until requested
 // outSize length. Provides a way to skip bytes until the offset.
-func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
+func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
 	// Offset and out size cannot be negative.
 	if offset < 0 || length < 0 {
-		return 0, errors.Trace(errUnexpected)
+		logger.LogIf(ctx, errUnexpected)
+		return 0, errUnexpected
 	}
 
 	// Do we have enough blocks?
 	if len(enBlocks) < dataBlocks {
-		return 0, errors.Trace(reedsolomon.ErrTooFewShards)
+		logger.LogIf(ctx, reedsolomon.ErrTooFewShards)
+		return 0, reedsolomon.ErrTooFewShards
 	}
 
 	// Do we have enough data?
 	if int64(getDataBlockLen(enBlocks, dataBlocks)) < length {
-		return 0, errors.Trace(reedsolomon.ErrShortData)
+		logger.LogIf(ctx, reedsolomon.ErrShortData)
+		return 0, reedsolomon.ErrShortData
 	}
 
 	// Counter to decrement total left to write.
@@ -77,7 +81,8 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
 		if write < int64(len(block)) {
 			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
 			if err != nil {
-				return 0, errors.Trace(err)
+				logger.LogIf(ctx, err)
+				return 0, err
 			}
 			totalWritten += n
 			break
@@ -85,7 +90,8 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
 		// Copy the block.
 		n, err := io.Copy(dst, bytes.NewReader(block))
 		if err != nil {
-			return 0, errors.Trace(err)
+			logger.LogIf(ctx, err)
+			return 0, err
 		}
 
 		// Decrement output size.

cmd/erasure.go

@@ -17,11 +17,12 @@
 package cmd
 
 import (
+	"context"
 	"crypto/subtle"
 	"hash"
 
 	"github.com/klauspost/reedsolomon"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // OfflineDisk represents an unavailable disk.
@@ -44,11 +45,12 @@ type ErasureStorage struct {
 
 // NewErasureStorage creates a new ErasureStorage. The storage erasure codes and protects all data written to
 // the disks.
-func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int, blockSize int64) (s ErasureStorage, err error) {
+func NewErasureStorage(ctx context.Context, disks []StorageAPI, dataBlocks, parityBlocks int, blockSize int64) (s ErasureStorage, err error) {
 	shardsize := (int(blockSize) + dataBlocks - 1) / dataBlocks
 	erasure, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(shardsize))
 	if err != nil {
-		return s, errors.Tracef("failed to create erasure coding: %v", err)
+		logger.LogIf(ctx, err)
+		return s, err
 	}
 	s = ErasureStorage{
 		disks:        make([]StorageAPI, len(disks)),
@@ -62,13 +64,15 @@ func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int, blockSi
 
 // ErasureEncode encodes the given data and returns the erasure-coded data.
 // It returns an error if the erasure coding failed.
-func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) {
+func (s *ErasureStorage) ErasureEncode(ctx context.Context, data []byte) ([][]byte, error) {
 	encoded, err := s.erasure.Split(data)
 	if err != nil {
-		return nil, errors.Tracef("failed to split data: %v", err)
+		logger.LogIf(ctx, err)
+		return nil, err
 	}
 	if err = s.erasure.Encode(encoded); err != nil {
-		return nil, errors.Tracef("failed to encode data: %v", err)
+		logger.LogIf(ctx, err)
+		return nil, err
 	}
 	return encoded, nil
 }
@@ -78,16 +82,17 @@ func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) {
 // It returns an error if the decoding failed.
 func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
 	if err := s.erasure.ReconstructData(data); err != nil {
-		return errors.Tracef("failed to reconstruct data: %v", err)
+		return err
 	}
 	return nil
 }
 
 // ErasureDecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
 // It returns an error if the decoding failed.
-func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(data [][]byte) error {
+func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
 	if err := s.erasure.Reconstruct(data); err != nil {
-		return errors.Tracef("failed to reconstruct data: %v", err)
+		logger.LogIf(ctx, err)
+		return err
 	}
 	return nil
 }

cmd/erasure_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"io"
 	"os"
@@ -52,11 +53,11 @@ func TestErasureDecode(t *testing.T) {
 		copy(buffer, data)
 
 		disks := make([]StorageAPI, test.dataBlocks+test.parityBlocks)
-		storage, err := NewErasureStorage(disks, test.dataBlocks, test.parityBlocks, blockSizeV1)
+		storage, err := NewErasureStorage(context.Background(), disks, test.dataBlocks, test.parityBlocks, blockSizeV1)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create erasure storage: %v", i, err)
 		}
-		encoded, err := storage.ErasureEncode(buffer)
+		encoded, err := storage.ErasureEncode(context.Background(), buffer)
 		if err != nil {
 			t.Fatalf("Test %d: failed to encode data: %v", i, err)
 		}
@@ -69,7 +70,7 @@ func TestErasureDecode(t *testing.T) {
 		}
 
 		if test.reconstructParity {
-			err = storage.ErasureDecodeDataAndParityBlocks(encoded)
+			err = storage.ErasureDecodeDataAndParityBlocks(context.Background(), encoded)
 		} else {
 			err = storage.ErasureDecodeDataBlocks(encoded)
 		}
@@ -98,7 +99,7 @@ func TestErasureDecode(t *testing.T) {
 			}
 
 			decodedData := new(bytes.Buffer)
-			if _, err = writeDataBlocks(decodedData, decoded, test.dataBlocks, 0, int64(len(data))); err != nil {
+			if _, err = writeDataBlocks(context.Background(), decodedData, decoded, test.dataBlocks, 0, int64(len(data))); err != nil {
 				t.Errorf("Test %d: failed to write data blocks: %v", i, err)
 			}
 			if !bytes.Equal(decodedData.Bytes(), data) {

cmd/format-disk-cache.go

@@ -17,13 +17,14 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"os"
 	"reflect"
 
-	errors2 "github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 const (
@@ -86,28 +87,19 @@ func newFormatCacheV1(drives []string) []*formatCacheV1 {
 	return formats
 }
 
-// Returns format.Cache.Version
-func formatCacheGetVersion(r io.ReadSeeker) (string, error) {
-	format := &formatCacheVersionDetect{}
-	if err := jsonLoad(r, format); err != nil {
-		return "", err
-	}
-	return format.Cache.Version, nil
-}
-
 // Creates a new cache format.json if unformatted.
 func createFormatCache(fsFormatPath string, format *formatCacheV1) error {
 	// open file using READ & WRITE permission
 	var file, err = os.OpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
-		return errors2.Trace(err)
+		return err
 	}
 	// Close the locked file upon return.
 	defer file.Close()
 
 	fi, err := file.Stat()
 	if err != nil {
-		return errors2.Trace(err)
+		return err
 	}
 	if fi.Size() != 0 {
 		// format.json already got created because of another minio process's createFormatCache()
@@ -118,7 +110,7 @@ func createFormatCache(fsFormatPath string, format *formatCacheV1) error {
 
 // This function creates a cache format file on disk and returns a slice
 // of format cache config
-func initFormatCache(drives []string) (formats []*formatCacheV1, err error) {
+func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV1, err error) {
 	nformats := newFormatCacheV1(drives)
 	for _, drive := range drives {
 		_, err = os.Stat(drive)
@@ -126,28 +118,36 @@ func initFormatCache(drives []string) (formats []*formatCacheV1, err error) {
 			continue
 		}
 		if !os.IsNotExist(err) {
+			logger.GetReqInfo(ctx).AppendTags("drive", drive)
+			logger.LogIf(ctx, err)
 			return nil, err
 		}
 		if err = os.Mkdir(drive, 0777); err != nil {
+			logger.GetReqInfo(ctx).AppendTags("drive", drive)
+			logger.LogIf(ctx, err)
 			return nil, err
 		}
 	}
 	for i, drive := range drives {
 		if err = os.Mkdir(pathJoin(drive, minioMetaBucket), 0777); err != nil {
 			if !os.IsExist(err) {
+				logger.GetReqInfo(ctx).AppendTags("drive", drive)
+				logger.LogIf(ctx, err)
 				return nil, err
 			}
 		}
 		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
 		// Fresh disk - create format.json for this cfs
 		if err = createFormatCache(cacheFormatPath, nformats[i]); err != nil {
+			logger.GetReqInfo(ctx).AppendTags("drive", drive)
+			logger.LogIf(ctx, err)
 			return nil, err
 		}
 	}
 	return nformats, nil
 }
 
-func loadFormatCache(drives []string) ([]*formatCacheV1, error) {
+func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV1, error) {
 	formats := make([]*formatCacheV1, len(drives))
 	for i, drive := range drives {
 		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
@@ -156,6 +156,7 @@ func loadFormatCache(drives []string) ([]*formatCacheV1, error) {
 			if os.IsNotExist(err) {
 				continue
 			}
+			logger.LogIf(ctx, err)
 			return nil, err
 		}
 		defer f.Close()
@@ -268,7 +269,7 @@ func findCacheDiskIndex(disk string, disks []string) int {
 }
 
 // validate whether cache drives order has changed
-func validateCacheFormats(formats []*formatCacheV1) error {
+func validateCacheFormats(ctx context.Context, formats []*formatCacheV1) error {
 	count := 0
 	for _, format := range formats {
 		if format == nil {
@@ -279,12 +280,16 @@ func validateCacheFormats(formats []*formatCacheV1) error {
 		return errors.New("Cache format files missing on all drives")
 	}
 	if _, err := checkFormatCacheValues(formats); err != nil {
+		logger.LogIf(ctx, err)
 		return err
 	}
 	if err := checkCacheDisksSliceConsistency(formats); err != nil {
+		logger.LogIf(ctx, err)
 		return err
 	}
-	return checkCacheDiskConsistency(formats)
+	err := checkCacheDiskConsistency(formats)
+	logger.LogIf(ctx, err)
+	return err
 }
 
 // return true if all of the list of cache drives are
@@ -303,16 +308,16 @@ func cacheDrivesUnformatted(drives []string) bool {
 // create format.json for each cache drive if fresh disk or load format from disk
 // Then validate the format for all drives in the cache to ensure order
 // of cache drives has not changed.
-func loadAndValidateCacheFormat(drives []string) (formats []*formatCacheV1, err error) {
+func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV1, err error) {
 	if cacheDrivesUnformatted(drives) {
-		formats, err = initFormatCache(drives)
+		formats, err = initFormatCache(ctx, drives)
 	} else {
-		formats, err = loadFormatCache(drives)
+		formats, err = loadFormatCache(ctx, drives)
 	}
 	if err != nil {
 		return nil, err
 	}
-	if err = validateCacheFormats(formats); err != nil {
+	if err = validateCacheFormats(ctx, formats); err != nil {
 		return nil, err
 	}
 	return formats, nil