Update federation target to etcd/clientv3 (#6119)

With CoreDNS now supporting etcdv3 as the DNS backend, we
can update our federation target to etcdv3. Users will now be
able to use etcdv3 server as the federation backbone.

Minio will update bucket data to etcdv3 and CoreDNS can pick
that data up and serve it as bucket style DNS path.

.travis.yml

@@ -8,27 +8,35 @@ dist: trusty
 
 language: go
 
-os:
-- linux
-
-env:
-- ARCH=x86_64
+matrix:
+  include:
+    - os: linux
+      env:
+        - ARCH=x86_64
+      go: 1.10.1
+      script:
+        - make
+        - diff -au <(gofmt -s -d cmd) <(printf "")
+        - diff -au <(gofmt -s -d pkg) <(printf "")
+        - make test GOFLAGS="-timeout 15m -race -v"
+        - make coverage
+        - node --version
+        - cd browser && yarn && yarn test && cd ..
+    - os: linux-ppc64le
+      env:
+        - ARCH=ppc64le
+      go: 1.10.1
+      script:
+        - make
+        - diff -au <(gofmt -s -d cmd) <(printf "")
+        - diff -au <(gofmt -s -d pkg) <(printf "")
+        - make test GOFLAGS="-timeout 15m -v"
+        - make coverage
+        - node --version
+        - cd browser && yarn && yarn test && cd ..
 
 before_install:
 - nvm install stable
 
-script:
-## Run all the tests
-- make
-- diff -au <(gofmt -s -d cmd) <(printf "")
-- diff -au <(gofmt -s -d pkg) <(printf "")
-- make test GOFLAGS="-timeout 15m -race -v"
-- make coverage
-- node --version
-- cd browser && yarn && yarn test && cd ..
-
 after_success:
-- bash <(curl -s https://codecov.io/bash)
-
-go:
-- 1.9.4
+- bash <(curl -s https://codecov.io/bash)

CONTRIBUTING.md

@@ -6,7 +6,7 @@
 
 Start by forking the Minio GitHub repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details:
 
-### Setup your Minio Github Repository
+### Setup your Minio GitHub Repository
 Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL of your Minio fork (you will need it for the `git clone` command below).
 
 ```sh

Dockerfile

@@ -1,6 +1,6 @@
-FROM golang:1.9.4-alpine3.6
+FROM golang:1.10.1-alpine3.7
 
-MAINTAINER Minio Inc <dev@minio.io>
+LABEL maintainer="Minio Inc <dev@minio.io>"
 
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin

Dockerfile.dev

@@ -1,6 +1,6 @@
-FROM golang:1.9.4-alpine3.6
+FROM golang:1.10.1-alpine3.7
 
-MAINTAINER Minio Inc <dev@minio.io>
+LABEL maintainer="Minio Inc <dev@minio.io>"
 
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin

Dockerfile.release

@@ -1,7 +1,7 @@
-FROM alpine:3.6
+FROM alpine:3.7
+
+LABEL maintainer="Minio Inc <dev@minio.io>"
 
-MAINTAINER Minio Inc <dev@minio.io>
- 
 COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 
 ENV MINIO_UPDATE off
@@ -27,4 +27,3 @@ HEALTHCHECK --interval=30s --timeout=5s \
     CMD /usr/bin/healthcheck.sh
 
 CMD ["minio"]
-

MAINTAINERS.md

@@ -1,6 +1,6 @@
 # For maintainers only
 
-### Setup your minio Github Repository
+### Setup your minio GitHub Repository
 
 Fork [minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository.
 ```bash

Makefile

@@ -50,15 +50,17 @@ deadcode:
 	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
 
 spelling:
-	@${GOPATH}/bin/misspell -error `find cmd/`
-	@${GOPATH}/bin/misspell -error `find pkg/`
-	@${GOPATH}/bin/misspell -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find cmd/`
+	@${GOPATH}/bin/misspell -locale US -error `find pkg/`
+	@${GOPATH}/bin/misspell -locale US -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
+	@${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build
 	@echo "Running unit tests"
-	@go test $(GOFLAGS) ./...
+	@go test $(GOFLAGS) -tags kqueue ./...
 	@echo "Verifying build"
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
@@ -69,7 +71,7 @@ coverage: build
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@CGO_ENABLED=0 go build --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
+	@CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
 
 pkg-add:
 	@echo "Adding new package $(PKG)"
@@ -89,7 +91,7 @@ pkg-list:
 # Builds minio and installs it to $GOPATH/bin.
 install: build
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
-	@cp $(PWD)/minio $(GOPATH)/bin/minio
+	@mkdir -p $(GOPATH)/bin && cp $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
 clean:

README.md

@@ -101,7 +101,7 @@ iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
 service iptables restart
 ```
 
-### ufw in Debian
+### ufw
 
 For hosts with ufw enabled (Debian based distros), you can use `ufw` command to allow traffic to specific ports. Use below command to allow access to port 9000
 
@@ -115,10 +115,30 @@ Below command enables all incoming traffic to ports ranging from 9000 to 9010.
 ufw allow 9000:9010/tcp
 ```
 
+### firewall-cmd
+
+For hosts with firewall-cmd enabled (CentOS), you can use `firewall-cmd` command to allow traffic to specific ports. Use below commands to allow access to port 9000
+
+```sh
+firewall-cmd --get-active-zones
+```
+
+This command gets the active zone(s). Now, apply port rules to the relevant zones returned above. For example if the zone is `public`, use
+
+```sh
+firewall-cmd --zone=public --add-port=9000/tcp --permanent
+```
+
+Note that `permanent` makes sure the rules are persistent across firewall start, restart or reload. Finally reload the firewall for changes to take effect.
+
+```sh
+firewall-cmd --reload
+```
+
 ## Test using Minio Browser
 Minio Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully.
 
-![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true)
+![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.png?raw=true)
 
 ## Test using Minio Client `mc`
 `mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the Minio Client [Quickstart Guide](https://docs.minio.io/docs/minio-client-quickstart-guide) for further instructions.

README_zh_CN.md

@@ -84,7 +84,7 @@ go get -u github.com/minio/minio
 ## 使用Minio浏览器进行验证
 安装后使用浏览器访问[http://127.0.0.1:9000](http://127.0.0.1:9000)，如果可以访问，则表示minio已经安装成功。
 
-![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true)
+![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.png?raw=true)
 
 ## 使用Minio客户端 `mc`进行验证
 `mc` 提供了一些UNIX常用命令的替代品，像ls, cat, cp, mirror, diff这些。 它支持文件系统和亚马逊S3云存储服务。 更多信息请参考 [mc快速入门](https://docs.minio.io/docs/minio-client-quickstart-guide) 。

appveyor.yml

@@ -12,7 +12,7 @@ clone_folder: c:\gopath\src\github.com\minio\minio
 # Environment variables
 environment:
   GOPATH: c:\gopath
-  GOROOT: c:\go19
+  GOROOT: c:\go
 
 # scripts that run after cloning repository
 install:

browser/app/js/browser/BrowserDropdown.js

@@ -89,7 +89,7 @@ export class BrowserDropdown extends React.Component {
           <Dropdown.Menu className="dropdown-menu-right">
             <li>
               <a target="_blank" href="https://github.com/minio/minio">
-                Github <i className="fa fa-github" />
+                GitHub <i className="fa fa-github" />
               </a>
             </li>
             <li>

browser/app/js/browser/ChangePasswordModal.js

@@ -42,7 +42,7 @@ export class ChangePasswordModal extends React.Component {
     const { serverInfo } = this.props
 
     // Check environment variables first.
-    if (serverInfo.info.isEnvCreds) {
+    if (serverInfo.info.isEnvCreds || serverInfo.info.isWorm) {
       this.setState({
         accessKey: "xxxxxxxxx",
         secretKey: "xxxxxxxxx",

browser/app/js/browser/Login.js

@@ -25,14 +25,35 @@ import web from "../web"
 import { Redirect } from "react-router-dom"
 
 export class Login extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      accessKey: "",
+      secretKey: ""
+    }
+  }
+
+  // Handle field changes
+  accessKeyChange(e) {
+    this.setState({
+      accessKey: e.target.value
+    })
+  }
+
+  secretKeyChange(e) {
+    this.setState({
+      secretKey: e.target.value
+    })
+  }
+
   handleSubmit(event) {
     event.preventDefault()
     const { showAlert, history } = this.props
     let message = ""
-    if (!document.getElementById("accessKey").value) {
+    if (this.state.accessKey === "") {
       message = "Access Key cannot be empty"
     }
-    if (!document.getElementById("secretKey").value) {
+    if (this.state.secretKey === "") {
       message = "Secret Key cannot be empty"
     }
     if (message) {
@@ -41,8 +62,8 @@ export class Login extends React.Component {
     }
     web
       .Login({
-        username: document.getElementById("accessKey").value,
-        password: document.getElementById("secretKey").value
+        username: this.state.accessKey,
+        password: this.state.secretKey
       })
       .then(res => {
         history.push("/")
@@ -77,6 +98,8 @@ export class Login extends React.Component {
         <div className="l-wrap">
           <form onSubmit={this.handleSubmit.bind(this)}>
             <InputGroup
+              value={this.state.accessKey}
+              onChange={this.accessKeyChange.bind(this)}
               className="ig-dark"
               label="Access Key"
               id="accessKey"
@@ -87,6 +110,8 @@ export class Login extends React.Component {
               autoComplete="username"
             />
             <InputGroup
+              value={this.state.secretKey}
+              onChange={this.secretKeyChange.bind(this)}
               className="ig-dark"
               label="Secret Key"
               id="secretKey"

browser/app/js/browser/MainActions.js

@@ -33,7 +33,11 @@ export const MainActions = ({
   )
   const onFileUpload = e => {
     e.preventDefault()
-    uploadFile(e.target.files[0])
+    let files = e.target.files
+    let filesToUploadCount = files.length
+    for (let i = 0; i < filesToUploadCount; i++) {
+      uploadFile(files.item(i))
+    }
     e.target.value = null
   }
 
@@ -55,6 +59,7 @@ export const MainActions = ({
                 onChange={onFileUpload}
                 style={{ display: "none" }}
                 id="file-input"
+                multiple={true}
               />
               <label htmlFor="file-input">
                 {" "}

browser/app/js/browser/StorageInfo.js

@@ -25,23 +25,16 @@ export class StorageInfo extends React.Component {
     fetchStorageInfo()
   }
   render() {
-    const { total, free } = this.props.storageInfo
-    const used = total - free
-    const usedPercent = used / total * 100 + "%"
-    const freePercent = free * 100 / total
+    const { used } = this.props.storageInfo
     return (
-      <div className="feh-usage">
+      <div className="feh-used">
         <div className="fehu-chart">
-          <div style={{ width: usedPercent }} />
+          <div style={{ width: 0 }} />
         </div>
         <ul>
           <li>
             <span>Used: </span>
-            {humanize.filesize(total - free)}
-          </li>
-          <li className="pull-right">
-            <span>Free: </span>
-            {humanize.filesize(total - used)}
+            {humanize.filesize(used)}
           </li>
         </ul>
       </div>

browser/app/js/browser/__tests__/Login.test.js

@@ -60,20 +60,25 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
     // case where both keys are empty - displays the second warning
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
 
     // case where access key is empty
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Access Key cannot be empty")
 
     // case where secret key is empty
-    document.getElementById("accessKey").value = "accessKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: ""
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
   })
@@ -85,11 +90,12 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
-    document.getElementById("accessKey").value = "accessKey"
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(web.Login).toHaveBeenCalledWith({
       "username": "accessKey", 

browser/app/js/browser/__tests__/MainActions.test.js

@@ -63,11 +63,20 @@ describe("MainActions", () => {
   it("should call uploadFile when a file is selected for upload", () => {
     const uploadFile = jest.fn()
     const wrapper = shallow(<MainActions uploadFile={uploadFile} />)
-    const file = new Blob(["file content"], { type: "text/plain" })
-    wrapper.find("#file-input").simulate("change", {
+    const files = [new Blob(["file content"], { type: "text/plain" })]
+    const input = wrapper.find("#file-input")
+    const event = {
       preventDefault: jest.fn(),
-      target: { files: [file] }
-    })
-    expect(uploadFile).toHaveBeenCalledWith(file)
+      target: {
+        files: {
+          length: files.length,
+          item: function(index) {
+            return files[index]
+          }
+        }
+      }
+    }
+    input.simulate("change", event)
+    expect(uploadFile).toHaveBeenCalledWith(files[0])
   })
 })

browser/app/js/browser/__tests__/StorageInfo.test.js

@@ -22,7 +22,7 @@ describe("StorageInfo", () => {
   it("should render without crashing", () => {
     shallow(
       <StorageInfo
-        storageInfo={{ total: 100, free: 60 }}
+        storageInfo={{ used: 60 }}
         fetchStorageInfo={jest.fn()}
       />
     )
@@ -32,7 +32,7 @@ describe("StorageInfo", () => {
     const fetchStorageInfo = jest.fn()
     shallow(
       <StorageInfo
-        storageInfo={{ total: 100, free: 60 }}
+        storageInfo={{ used: 60 }}
         fetchStorageInfo={fetchStorageInfo}
       />
     )

browser/app/js/browser/__tests__/actions.test.js

@@ -20,7 +20,7 @@ import * as actionsCommon from "../actions"
 
 jest.mock("../../web", () => ({
   StorageInfo: jest.fn(() => {
-    return Promise.resolve({ storageInfo: { Total: 100, Free: 60 } })
+    return Promise.resolve({ storageInfo: { Used: 60 } })
   }),
   ServerInfo: jest.fn(() => {
     return Promise.resolve({
@@ -40,7 +40,7 @@ describe("Common actions", () => {
   it("creates common/SET_STORAGE_INFO after fetching the storage details ", () => {
     const store = mockStore()
     const expectedActions = [
-      { type: "common/SET_STORAGE_INFO", storageInfo: { total: 100, free: 60 } }
+      { type: "common/SET_STORAGE_INFO", storageInfo: { used: 60 } }
     ]
     return store.dispatch(actionsCommon.fetchStorageInfo()).then(() => {
       const actions = store.getActions()

browser/app/js/browser/actions.js

@@ -34,7 +34,7 @@ export const fetchStorageInfo = () => {
     return web.StorageInfo().then(res => {
       const storageInfo = {
         total: res.storageInfo.Total,
-        free: res.storageInfo.Free
+        used:  res.storageInfo.Used
       }
       dispatch(setStorageInfo(storageInfo))
     })

browser/app/js/objects/ObjectContainer.js

@@ -37,7 +37,7 @@ export const ObjectContainer = ({
   if (checkedObjectsCount == 0) {
     props.actionButtons = <ObjectActions object={object} />
   }
-  return <ObjectItem {...props} onClick={() => downloadObject(object.name)} />
+  return <ObjectItem {...props} />
 }
 
 const mapStateToProps = state => {
@@ -46,10 +46,4 @@ const mapStateToProps = state => {
   }
 }
 
-const mapDispatchToProps = dispatch => {
-  return {
-    downloadObject: object => dispatch(actionsObjects.downloadObject(object))
-  }
-}
-
-export default connect(mapStateToProps, mapDispatchToProps)(ObjectContainer)
+export default connect(mapStateToProps)(ObjectContainer)

browser/app/js/objects/ObjectItem.js

@@ -54,7 +54,9 @@ export const ObjectItem = ({
           href="#"
           onClick={e => {
             e.preventDefault()
-            onClick()
+            if (onClick) {
+              onClick()
+            }
           }}
         >
           {name}

browser/app/js/objects/ObjectsBulkActions.js

@@ -28,6 +28,15 @@ export class ObjectsBulkActions extends React.Component {
       showDeleteConfirmation: false
     }
   }
+  handleDownload() {
+    const { checkedObjects, clearChecked, downloadChecked, downloadObject } = this.props
+    if (checkedObjects.length === 1 && !checkedObjects[0].endsWith('/')) {
+      downloadObject(checkedObjects[0])
+      clearChecked()
+    } else {
+      downloadChecked()
+    }
+  }
   deleteChecked() {
     const { deleteChecked } = this.props
     deleteChecked()
@@ -39,24 +48,27 @@ export class ObjectsBulkActions extends React.Component {
     })
   }
   render() {
-    const { checkedObjectsCount, downloadChecked, clearChecked } = this.props
+    const { checkedObjects, clearChecked } = this.props
     return (
       <div
         className={
           "list-actions" +
           classNames({
-            " list-actions-toggled": checkedObjectsCount > 0
+            " list-actions-toggled": checkedObjects.length > 0
           })
         }
       >
         <span className="la-label">
-          <i className="fa fa-check-circle" /> {checkedObjectsCount} Objects
+          <i className="fa fa-check-circle" /> {checkedObjects.length}
+          {checkedObjects.length === 1 ? " Object " : " Objects "}
           selected
         </span>
         <span className="la-actions pull-right">
-          <button id="download-checked" onClick={downloadChecked}>
+          <button id="download-checked" onClick={this.handleDownload.bind(this)}>
             {" "}
-            Download all as zip{" "}
+            Download
+            {(checkedObjects.length === 1 && !checkedObjects[0].endsWith('/')) ? 
+            " object" : " all as zip" }{" "}
           </button>
         </span>
         <span className="la-actions pull-right">
@@ -86,13 +98,16 @@ export class ObjectsBulkActions extends React.Component {
 
 const mapStateToProps = state => {
   return {
-    checkedObjectsCount: getCheckedList(state).length
+    checkedObjects: getCheckedList(state)
   }
 }
 
 const mapDispatchToProps = dispatch => {
   return {
+    downloadObject: object => dispatch(actions.downloadObject(object)),
     downloadChecked: () => dispatch(actions.downloadCheckedObjects()),
+    downloadObject: object => dispatch(actions.downloadObject(object)),
+    resetCheckedList: () => dispatch(actions.resetCheckedList()),
     clearChecked: () => dispatch(actions.resetCheckedList()),
     deleteChecked: () => dispatch(actions.deleteCheckedObjects())
   }

browser/app/js/objects/__tests__/ObjectItem.test.js

@@ -28,9 +28,16 @@ describe("ObjectItem", () => {
     expect(wrapper.prop("data-type")).toBe("image")
   })
 
-  it("should call onClick when the object isclicked", () => {
+  it("shouldn't call onClick when the object isclicked", () => {
     const onClick = jest.fn()
-    const wrapper = shallow(<ObjectItem name={"test"} onClick={onClick} />)
+    const wrapper = shallow(<ObjectItem name={"test"} />)
+    wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
+    expect(onClick).not.toHaveBeenCalled()
+  })
+
+  it("should call onClick when the folder isclicked", () => {
+    const onClick = jest.fn()
+    const wrapper = shallow(<ObjectItem name={"test/"} onClick={onClick} />)
     wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
     expect(onClick).toHaveBeenCalled()
   })

browser/app/js/objects/__tests__/ObjectsBulkActions.test.js

@@ -20,19 +20,45 @@ import { ObjectsBulkActions } from "../ObjectsBulkActions"
 
 describe("ObjectsBulkActions", () => {
   it("should render without crashing", () => {
-    shallow(<ObjectsBulkActions checkedObjectsCount={0} />)
+    shallow(<ObjectsBulkActions checkedObjects={[]} />)
   })
 
   it("should show actions when checkObjectsCount is more than 0", () => {
-    const wrapper = shallow(<ObjectsBulkActions checkedObjectsCount={1} />)
+    const wrapper = shallow(<ObjectsBulkActions checkedObjects={["test"]} />)
     expect(wrapper.hasClass("list-actions-toggled")).toBeTruthy()
   })
 
-  it("should call downloadChecked when download button is clicked", () => {
+  it("should call downloadObject when single object is selected and download button is clicked", () => {
+    const downloadObject = jest.fn()
+    const clearChecked = jest.fn()
+    const wrapper = shallow(
+      <ObjectsBulkActions
+        checkedObjects={["test"]}
+        downloadObject={downloadObject}
+        clearChecked={clearChecked}
+      />
+    )
+    wrapper.find("#download-checked").simulate("click")
+    expect(downloadObject).toHaveBeenCalled()
+  })
+
+  it("should call downloadChecked when a folder is selected and download button is clicked", () => {
     const downloadChecked = jest.fn()
     const wrapper = shallow(
       <ObjectsBulkActions
-        checkedObjectsCount={1}
+        checkedObjects={["test/"]}
+        downloadChecked={downloadChecked}
+      />
+    )
+    wrapper.find("#download-checked").simulate("click")
+    expect(downloadChecked).toHaveBeenCalled()
+  })
+
+  it("should call downloadChecked when multiple objects are selected and download button is clicked", () => {
+    const downloadChecked = jest.fn()
+    const wrapper = shallow(
+      <ObjectsBulkActions
+        checkedObjects={["test1", "test2"]}
         downloadChecked={downloadChecked}
       />
     )
@@ -43,14 +69,14 @@ describe("ObjectsBulkActions", () => {
   it("should call clearChecked when close button is clicked", () => {
     const clearChecked = jest.fn()
     const wrapper = shallow(
-      <ObjectsBulkActions checkedObjectsCount={1} clearChecked={clearChecked} />
+      <ObjectsBulkActions checkedObjects={["test"]} clearChecked={clearChecked} />
     )
     wrapper.find("#close-bulk-actions").simulate("click")
     expect(clearChecked).toHaveBeenCalled()
   })
 
   it("shoud show DeleteObjectConfirmModal when delete-checked button is clicked", () => {
-    const wrapper = shallow(<ObjectsBulkActions checkedObjectsCount={1} />)
+    const wrapper = shallow(<ObjectsBulkActions checkedObjects={["test"]} />)
     wrapper.find("#delete-checked").simulate("click")
     wrapper.update()
     expect(wrapper.find("DeleteObjectConfirmModal").length).toBe(1)
@@ -60,7 +86,7 @@ describe("ObjectsBulkActions", () => {
     const deleteChecked = jest.fn()
     const wrapper = shallow(
       <ObjectsBulkActions
-        checkedObjectsCount={1}
+        checkedObjects={["test"]}
         deleteChecked={deleteChecked}
       />
     )

browser/app/less/inc/form.less

@@ -87,6 +87,7 @@ select.form-control {
     z-index: 1;
     border-bottom: 1px solid #eee;
     color: #32393F;
+    box-shadow: none;
     font-size: 13px;
 
 

browser/app/less/inc/header.less

@@ -44,9 +44,9 @@
 
 
 /*--------------------------
-    Disk usage
+    Disk used
 ----------------------------*/
-.feh-usage {
+.feh-used {
     margin-top: 12px;
     max-width: 285px;
 

buildscripts/checkdeps.sh

@@ -21,7 +21,7 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GO_VERSION="1.9.4"
+    GO_VERSION="1.10.1"
     OSX_VERSION="10.8"
     KNAME=$(uname -s)
     ARCH=$(uname -m)
@@ -34,7 +34,7 @@ _init() {
 
 ## FIXME:
 ## In OSX, 'readlink -f' option does not exist, hence
-## we have our own readlink -f behaviour here.
+## we have our own readlink -f behavior here.
 ## Once OSX has the option, below function is good enough.
 ## 
 ## readlink() {
@@ -89,11 +89,11 @@ check_minimum_version() {
 
 assert_is_supported_arch() {
     case "${ARCH}" in
-        x86_64 | amd64 | aarch64 | arm* )
+        x86_64 | amd64 | aarch64 | ppc64le | arm* )
             return
             ;;
         *)
-            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, arm*]"
+            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*]"
             exit 1
     esac
 }

buildscripts/gen-ldflags.go

@@ -33,6 +33,7 @@ func genLDFlags(version string) string {
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
 	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
+	ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT")
 	return ldflagsStr
 }
 

cmd/acl-handlers.go

@@ -0,0 +1,145 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/policy"
+)
+
+// Data types used for returning dummy access control
+// policy XML, these variables shouldn't be used elsewhere
+// they are only defined to be used in this file alone.
+type grantee struct {
+	XMLNS       string `xml:"xmlns:xsi,attr"`
+	XMLXSI      string `xml:"xsi:type,attr"`
+	Type        string `xml:"Type"`
+	ID          string `xml:"ID,omitempty"`
+	DisplayName string `xml:"DisplayName,omitempty"`
+	URI         string `xml:"URI,omitempty"`
+}
+
+type grant struct {
+	Grantee    grantee `xml:"Grantee"`
+	Permission string  `xml:"Permission"`
+}
+
+type accessControlPolicy struct {
+	XMLName           xml.Name `xml:"AccessControlPolicy"`
+	Owner             Owner    `xml:"Owner"`
+	AccessControlList struct {
+		Grants []grant `xml:"Grant"`
+	} `xml:"AccessControlList"`
+}
+
+// GetBucketACLHandler - GET Bucket ACL
+// -----------------
+// This operation uses the ACL
+// subresource to return the ACL of a specified bucket.
+func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, "GetBucketACL")
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Allow getBucketACL if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
+	// Before proceeding validate if bucket exists.
+	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	acl := &accessControlPolicy{}
+	acl.AccessControlList.Grants = append(acl.AccessControlList.Grants, grant{
+		Grantee: grantee{
+			XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
+			XMLXSI: "CanonicalUser",
+			Type:   "CanonicalUser",
+		},
+		Permission: "FULL_CONTROL",
+	})
+	if err := xml.NewEncoder(w).Encode(acl); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// GetObjectACLHandler - GET Object ACL
+// -----------------
+// This operation uses the ACL
+// subresource to return the ACL of a specified object.
+func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, "GetObjectACL")
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Allow getObjectACL if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
+	// Before proceeding validate if object exists.
+	_, err := objAPI.GetObjectInfo(ctx, bucket, object)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	acl := &accessControlPolicy{}
+	acl.AccessControlList.Grants = append(acl.AccessControlList.Grants, grant{
+		Grantee: grantee{
+			XMLNS:  "http://www.w3.org/2001/XMLSchema-instance",
+			XMLXSI: "CanonicalUser",
+			Type:   "CanonicalUser",
+		},
+		Permission: "FULL_CONTROL",
+	})
+	if err := xml.NewEncoder(w).Encode(acl); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}

cmd/admin-handlers.go

@@ -18,7 +18,9 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -27,9 +29,11 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/quick"
 )
 
 const (
@@ -62,16 +66,16 @@ var (
 // -----------
 // Returns Administration API version
 func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
-		writeErrorResponse(w, adminAPIErr, r.URL)
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
 	}
 
 	jsonBytes, err := json.Marshal(adminAPIVersionInfo)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to marshal Admin API Version to JSON.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -82,7 +86,7 @@ func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request)
 // ----------
 // Returns server version and uptime.
 func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -99,7 +103,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 	uptime, err := getPeerUptimes(globalAdminPeers)
 	if err != nil {
 		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
-		errorIf(err, "Possibly failed to get uptime from majority of servers.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -113,7 +117,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 	jsonBytes, err := json.Marshal(serverStatus)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to marshal storage info into json.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 	// Reply with storage information (across nodes in a
@@ -127,7 +131,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 // Restarts/Stops minio server gracefully. In a distributed setup,
 // restarts all the servers in the cluster.
 func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -136,7 +140,7 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 	var sa madmin.ServiceAction
 	err := json.NewDecoder(r.Body).Decode(&sa)
 	if err != nil {
-		errorIf(err, "Error parsing body JSON")
+		logger.LogIf(context.Background(), err)
 		writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
 		return
 	}
@@ -149,7 +153,7 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 		serviceSig = serviceStop
 	default:
 		writeErrorResponseJSON(w, ErrMalformedPOSTRequest, r.URL)
-		errorIf(err, "Invalid service action received")
+		logger.LogIf(context.Background(), errors.New("Invalid service action received"))
 		return
 	}
 
@@ -219,7 +223,9 @@ type ServerInfo struct {
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	// Authenticate request
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+
+	// Setting the region as empty so as the mc server info command is irrespective to the region.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -241,9 +247,11 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 			// Initialize server info at index
 			reply[idx] = ServerInfo{Addr: peer.addr}
 
-			serverInfoData, err := peer.cmdRunner.ServerInfoData()
+			serverInfoData, err := peer.cmdRunner.ServerInfo()
 			if err != nil {
-				errorIf(err, "Unable to get server info from %s.", peer.addr)
+				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peer.addr)
+				ctx := logger.SetReqInfo(context.Background(), reqInfo)
+				logger.LogIf(ctx, err)
 				reply[idx].Error = err.Error()
 				return
 			}
@@ -258,7 +266,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	jsonBytes, err := json.Marshal(reply)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to marshal storage info into json.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -292,7 +300,7 @@ func validateLockQueryParams(vars url.Values) (string, string, time.Duration,
 	}
 	duration, err := time.ParseDuration(olderThanStr)
 	if err != nil {
-		errorIf(err, "Failed to parse duration passed as query value.")
+		logger.LogIf(context.Background(), err)
 		return "", "", time.Duration(0), ErrInvalidDuration
 	}
 
@@ -306,7 +314,7 @@ func validateLockQueryParams(vars url.Values) (string, string, time.Duration,
 // Lists locks held on a given bucket, prefix and duration it was held for.
 func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Request) {
 
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -325,7 +333,7 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques
 		duration)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to fetch lock information from remote nodes.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -333,7 +341,7 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques
 	jsonBytes, err := json.Marshal(volLocks)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to marshal lock information into json.")
+		logger.LogIf(context.Background(), err)
 		return
 	}
 
@@ -350,7 +358,14 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "ClearLocks")
 
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	// Get object layer instance.
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -369,7 +384,7 @@ func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Reque
 		duration)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to fetch lock information from remote nodes.")
+		logger.LogIf(ctx, err)
 		return
 	}
 
@@ -377,11 +392,11 @@ func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Reque
 	jsonBytes, err := json.Marshal(volLocks)
 	if err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		errorIf(err, "Failed to marshal lock information into json.")
+		logger.LogIf(ctx, err)
 		return
 	}
 
-	newObjectLayerFn().ClearLocks(ctx, volLocks)
+	objLayer.ClearLocks(ctx, volLocks)
 
 	// Reply with list of locks cleared, as json.
 	writeSuccessResponseJSON(w, jsonBytes)
@@ -425,7 +440,7 @@ func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
 	if clientToken == "" {
 		jerr := json.NewDecoder(r.Body).Decode(&hs)
 		if jerr != nil {
-			errorIf(jerr, "Error parsing body JSON")
+			logger.LogIf(context.Background(), jerr)
 			err = ErrRequestBodyParse
 			return
 		}
@@ -459,7 +474,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -487,6 +502,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	// after 10s unless a response item comes in
 	keepConnLive := func(w http.ResponseWriter, respCh chan healResp) {
 		ticker := time.NewTicker(time.Second * 10)
+		defer ticker.Stop()
 		started := false
 	forLoop:
 		for {
@@ -515,7 +531,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 				break forLoop
 			}
 		}
-		ticker.Stop()
 	}
 
 	// find number of disks in the setup
@@ -555,20 +570,13 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 // GetConfigHandler - GET /minio/admin/v1/config
 // Get config.json of this minio setup.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
-
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
 	}
 
-	// check if objectLayer is initialized, if not return.
-	if newObjectLayerFn() == nil {
-		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
-		return
-	}
-
 	// Take a read lock on minio/config.json. NB minio is a
 	// reserved bucket name and wouldn't conflict with normal
 	// object operations.
@@ -583,7 +591,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	// occurring on a quorum of the servers is returned.
 	configBytes, err := getPeerConfig(globalAdminPeers)
 	if err != nil {
-		errorIf(err, "Failed to get config from peers")
+		logger.LogIf(context.Background(), err)
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		return
 	}
@@ -655,6 +663,7 @@ func writeSetConfigResponse(w http.ResponseWriter, peers adminPeers,
 // SetConfigHandler - PUT /minio/admin/v1/config
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 
+	ctx := context.Background()
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -662,8 +671,14 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -674,12 +689,12 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	n, err := io.ReadFull(r.Body, configBuf)
 	if err == nil {
 		// More than maxConfigSize bytes were available
-		writeErrorResponse(w, ErrAdminConfigTooLarge, r.URL)
+		writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
 		return
 	}
 	if err != io.ErrUnexpectedEOF {
-		errorIf(err, "Failed to read config from request body.")
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
@@ -687,17 +702,17 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	// Validate JSON provided in the request body: check the
 	// client has not sent JSON objects with duplicate keys.
-	if err = checkDupJSONKeys(string(configBytes)); err != nil {
-		errorIf(err, "config contains duplicate JSON entries.")
-		writeErrorResponse(w, ErrAdminConfigBadJSON, r.URL)
+	if err = quick.CheckDuplicateKeys(string(configBytes)); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
 		return
 	}
 
 	var config serverConfig
 	err = json.Unmarshal(configBytes, &config)
 	if err != nil {
-		errorIf(err, "Failed to unmarshal JSON configuration", err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
@@ -718,7 +733,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	errs := writeTmpConfigPeers(globalAdminPeers, tmpFileName, configBytes)
 
 	// Check if the operation succeeded in quorum or more nodes.
-	rErr := reduceWriteQuorumErrs(errs, nil, len(globalAdminPeers)/2+1)
+	rErr := reduceWriteQuorumErrs(ctx, errs, nil, len(globalAdminPeers)/2+1)
 	if rErr != nil {
 		writeSetConfigResponse(w, globalAdminPeers, errs, false, r.URL)
 		return
@@ -736,7 +751,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	// Rename the temporary config file to config.json
 	errs = commitConfigPeers(globalAdminPeers, tmpFileName)
-	rErr = reduceWriteQuorumErrs(errs, nil, len(globalAdminPeers)/2+1)
+	rErr = reduceWriteQuorumErrs(ctx, errs, nil, len(globalAdminPeers)/2+1)
 	if rErr != nil {
 		writeSetConfigResponse(w, globalAdminPeers, errs, false, r.URL)
 		return
@@ -760,16 +775,16 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 	r *http.Request) {
 
 	// Authenticate request
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
-		writeErrorResponse(w, adminAPIErr, r.URL)
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
 	}
 
 	// Avoid setting new credentials when they are already passed
-	// by the environment.
-	if globalIsEnvCreds {
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+	// by the environment. Deny if WORM is enabled.
+	if globalIsEnvCreds || globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
 		return
 	}
 
@@ -777,14 +792,14 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 	var req madmin.SetCredsReq
 	err := json.NewDecoder(r.Body).Decode(&req)
 	if err != nil {
-		errorIf(err, "Error parsing body JSON")
+		logger.LogIf(context.Background(), err)
 		writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
 		return
 	}
 
 	creds, err := auth.CreateCredentials(req.AccessKey, req.SecretKey)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
@@ -801,19 +816,20 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 	globalServerConfigMu.Lock()
 	defer globalServerConfigMu.Unlock()
 
-	// Notify all other Minio peers to update credentials
-	updateErrs := updateCredsOnPeers(creds)
-	for peer, err := range updateErrs {
-		errorIf(err, "Unable to update credentials on peer %s.", peer)
-	}
-
 	// Update local credentials in memory.
 	globalServerConfig.SetCredential(creds)
-	if err = globalServerConfig.Save(); err != nil {
-		writeErrorResponse(w, ErrInternalError, r.URL)
+	if err = globalServerConfig.Save(getConfigFile()); err != nil {
+		writeErrorResponseJSON(w, ErrInternalError, r.URL)
 		return
 	}
 
+	// Notify all other Minio peers to update credentials
+	for host, err := range globalNotificationSys.SetCredentials(creds) {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", host.String())
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+	}
+
 	// At this stage, the operation is successful, return 200 OK
 	w.WriteHeader(http.StatusOK)
 }

cmd/admin-handlers_test.go

@@ -31,9 +31,8 @@ import (
 	"testing"
 	"time"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -141,7 +140,7 @@ type adminXLTestBed struct {
 	configPath string
 	xlDirs     []string
 	objLayer   ObjectLayer
-	mux        *router.Router
+	router     *mux.Router
 }
 
 // prepareAdminXLTestBed - helper function that setups a single-node
@@ -177,20 +176,20 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// Init global heal state
 	initAllHealState(globalIsXL)
 
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
-	if err != nil {
-		return nil, err
-	}
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
+
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
 
 	// Setup admin mgmt REST API handlers.
-	adminRouter := router.NewRouter()
+	adminRouter := mux.NewRouter()
 	registerAdminRouter(adminRouter)
 
 	return &adminXLTestBed{
 		configPath: rootPath,
 		xlDirs:     xlDirs,
 		objLayer:   objLayer,
-		mux:        adminRouter,
+		router:     adminRouter,
 	}, nil
 }
 
@@ -264,12 +263,13 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) {
 		return nil, nil, err
 	}
 	endpoints := mustGetNewEndpointList(xlDirs...)
-	format, err := waitForFormatXL(true, endpoints, 1, 16)
+	format, err := waitForFormatXL(context.Background(), true, endpoints, 1, 16)
 	if err != nil {
 		removeRoots(xlDirs)
 		return nil, nil, err
 	}
 
+	globalPolicySys = NewPolicySys()
 	objLayer, err := newXLSets(endpoints, format, 1, 16)
 	if err != nil {
 		return nil, nil, err
@@ -300,7 +300,7 @@ func TestAdminVersionHandler(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if http.StatusOK != rec.Code {
 		t.Errorf("Unexpected status code - got %d but expected %d",
 			rec.Code, http.StatusOK)
@@ -441,7 +441,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
-	// Setting up a go routine to simulate ServerMux's
+	// Setting up a go routine to simulate ServerRouter's
 	// handleServiceSignals for stop and restart commands.
 	if cmd == restartCmd {
 		go testServiceSignalReceiver(cmd, t)
@@ -460,7 +460,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 
 	if cmd == statusCmd {
 		expectedInfo := madmin.ServiceStatus{
@@ -544,7 +544,7 @@ func TestServiceSetCreds(t *testing.T) {
 		rec := httptest.NewRecorder()
 
 		// Execute request
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 
 		// Check if the http code response is expected
 		if rec.Code != testCase.ExpectedStatusCode {
@@ -637,7 +637,7 @@ func TestListLocksHandler(t *testing.T) {
 			t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if test.expectedStatus != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
 		}
@@ -704,7 +704,7 @@ func TestClearLocksHandler(t *testing.T) {
 			t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if test.expectedStatus != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
 		}
@@ -762,13 +762,13 @@ func buildAdminRequest(queryVal url.Values, method, path string,
 		"/minio/admin/v1"+path+"?"+queryVal.Encode(),
 		contentLength, bodySeeker)
 	if err != nil {
-		return nil, errors.Trace(err)
+		return nil, err
 	}
 
 	cred := globalServerConfig.GetCredential()
 	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 	if err != nil {
-		return nil, errors.Trace(err)
+		return nil, err
 	}
 
 	return req, nil
@@ -796,7 +796,7 @@ func TestGetConfigHandler(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
@@ -830,7 +830,7 @@ func TestSetConfigHandler(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
@@ -856,7 +856,7 @@ func TestSetConfigHandler(t *testing.T) {
 		}
 
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		respBody := string(rec.Body.Bytes())
 		if rec.Code != http.StatusBadRequest ||
 			!strings.Contains(respBody, "Configuration data provided exceeds the allowed maximum of") {
@@ -875,7 +875,7 @@ func TestSetConfigHandler(t *testing.T) {
 		}
 
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		respBody := string(rec.Body.Bytes())
 		if rec.Code != http.StatusBadRequest ||
 			!strings.Contains(respBody, "JSON configuration provided has objects with duplicate keys") {
@@ -905,7 +905,7 @@ func TestAdminServerInfo(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
@@ -927,9 +927,6 @@ func TestAdminServerInfo(t *testing.T) {
 		if serverInfo.Error != "" {
 			t.Errorf("Unexpected error = %v\n", serverInfo.Error)
 		}
-		if serverInfo.Data.StorageInfo.Free == 0 {
-			t.Error("Expected StorageInfo.Free to be non empty")
-		}
 		if serverInfo.Data.Properties.Region != globalMinioDefaultRegion {
 			t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, serverInfo.Data.Properties.Region)
 		}
@@ -1106,7 +1103,7 @@ func collectHealResults(t *testing.T, adminTestBed *adminXLTestBed, bucket,
 		}
 		req := mkHealStatusReq(t, bucket, prefix, clientToken)
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if http.StatusOK != rec.Code {
 			t.Errorf("Unexpected status code - got %d but expected %d",
 				rec.Code, http.StatusOK)
@@ -1152,7 +1149,7 @@ func TestHealStartNStatusHandler(t *testing.T) {
 	{
 		req := mkHealStartReq(t, bucketName, objName, healOpts)
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if http.StatusOK != rec.Code {
 			t.Errorf("Unexpected status code - got %d but expected %d",
 				rec.Code, http.StatusOK)
@@ -1172,7 +1169,7 @@ func TestHealStartNStatusHandler(t *testing.T) {
 		// test with an invalid client token
 		req := mkHealStatusReq(t, bucketName, objName, hss.ClientToken+hss.ClientToken)
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if rec.Code != http.StatusBadRequest {
 			t.Errorf("Unexpected status code")
 		}

cmd/admin-heal-ops.go

@@ -57,7 +57,7 @@ const (
 var (
 	errHealIdleTimeout      = fmt.Errorf("healing results were not consumed for too long")
 	errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
-	errHealStopSignalled    = fmt.Errorf("heal stop signalled")
+	errHealStopSignalled    = fmt.Errorf("heal stop signaled")
 
 	errFnHealFromAPIErr = func(err error) error {
 		errCode := toAPIErrorCode(err)
@@ -222,7 +222,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 		StartTime:     h.startTime,
 	})
 	if err != nil {
-		errorIf(err, "Failed to marshal heal result into json.")
+		logger.LogIf(context.Background(), err)
 		return nil, ErrInternalError, ""
 	}
 	return b, ErrNone, ""
@@ -270,7 +270,7 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 
 	jbytes, err := json.Marshal(h.currentStatus)
 	if err != nil {
-		errorIf(err, "Failed to marshal heal result into json.")
+		logger.LogIf(context.Background(), err)
 		return nil, ErrInternalError
 	}
 
@@ -301,7 +301,7 @@ type healSequence struct {
 	// current accumulated status of the heal sequence
 	currentStatus healSequenceStatus
 
-	// channel signalled by background routine when traversal has
+	// channel signaled by background routine when traversal has
 	// completed
 	traverseAndHealDoneCh chan error
 
@@ -321,7 +321,9 @@ type healSequence struct {
 func newHealSequence(bucket, objPrefix, clientAddr string,
 	numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence {
 
-	ctx := logger.SetContext(context.Background(), &logger.ReqInfo{clientAddr, "", "", "Heal", bucket, objPrefix, nil})
+	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
+	reqInfo.AppendTags("prefix", objPrefix)
+	ctx := logger.SetReqInfo(context.Background(), reqInfo)
 
 	return &healSequence{
 		bucket:        bucket,
@@ -439,7 +441,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 	h.currentStatus.updateLock.Unlock()
 
 	// This is a "safe" point for the heal sequence to quit if
-	// signalled externally.
+	// signaled externally.
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
@@ -537,11 +539,17 @@ func (h *healSequence) healDiskFormat() error {
 	}
 
 	res, err := objectAPI.HealFormat(h.ctx, h.settings.DryRun)
-	if err != nil {
+	// return any error, ignore error returned when disks have
+	// already healed.
+	if err != nil && err != errNoHealRequired {
 		return errFnHealFromAPIErr(err)
 	}
 
-	peersReInitFormat(globalAdminPeers, h.settings.DryRun)
+	// Healing succeeded notify the peers to reload format and re-initialize disks.
+	// We will not notify peers only if healing succeeded.
+	if err == nil {
+		peersReInitFormat(globalAdminPeers, h.settings.DryRun)
+	}
 
 	// Push format heal result
 	return h.pushHealResultItem(res)

cmd/admin-router.go

@@ -19,7 +19,7 @@ package cmd
 import (
 	"net/http"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 )
 
 const (
@@ -31,48 +31,48 @@ type adminAPIHandlers struct {
 }
 
 // registerAdminRouter - Add handler functions for each service REST API routes.
-func registerAdminRouter(mux *router.Router) {
+func registerAdminRouter(router *mux.Router) {
 
 	adminAPI := adminAPIHandlers{}
 	// Admin router
-	adminRouter := mux.NewRoute().PathPrefix(adminAPIPathPrefix).Subrouter()
+	adminRouter := router.PathPrefix(adminAPIPathPrefix).Subrouter()
 
 	// Version handler
-	adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(adminAPI.VersionHandler)
+	adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(httpTraceAll(adminAPI.VersionHandler))
 
 	adminV1Router := adminRouter.PathPrefix("/v1").Subrouter()
 
 	/// Service operations
 
 	// Service status
-	adminV1Router.Methods(http.MethodGet).Path("/service").HandlerFunc(adminAPI.ServiceStatusHandler)
+	adminV1Router.Methods(http.MethodGet).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceStatusHandler))
 
 	// Service restart and stop - TODO
-	adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(adminAPI.ServiceStopNRestartHandler)
+	adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(httpTraceAll(adminAPI.ServiceStopNRestartHandler))
 
 	// Info operations
-	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(adminAPI.ServerInfoHandler)
+	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
 
 	/// Lock operations
 
 	// List Locks
-	adminV1Router.Methods(http.MethodGet).Path("/locks").HandlerFunc(adminAPI.ListLocksHandler)
+	adminV1Router.Methods(http.MethodGet).Path("/locks").HandlerFunc(httpTraceAll(adminAPI.ListLocksHandler))
 	// Clear locks
-	adminV1Router.Methods(http.MethodDelete).Path("/locks").HandlerFunc(adminAPI.ClearLocksHandler)
+	adminV1Router.Methods(http.MethodDelete).Path("/locks").HandlerFunc(httpTraceAll(adminAPI.ClearLocksHandler))
 
 	/// Heal operations
 
 	// Heal processing endpoint.
-	adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(adminAPI.HealHandler)
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(adminAPI.HealHandler)
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(adminAPI.HealHandler)
+	adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
 	/// Config operations
 
 	// Update credentials
-	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(adminAPI.UpdateCredentialsHandler)
+	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceAll(adminAPI.UpdateCredentialsHandler))
 	// Get config
-	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(adminAPI.GetConfigHandler)
+	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceAll(adminAPI.GetConfigHandler))
 	// Set config
-	adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(adminAPI.SetConfigHandler)
+	adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceAll(adminAPI.SetConfigHandler))
 }

cmd/admin-rpc-client.go

@@ -18,39 +18,128 @@ package cmd
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"net"
-	"os"
-	"path"
-	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
-	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
-const (
-	// Admin service names
-	signalServiceRPC  = "Admin.SignalService"
-	reInitFormatRPC   = "Admin.ReInitFormat"
-	listLocksRPC      = "Admin.ListLocks"
-	serverInfoDataRPC = "Admin.ServerInfoData"
-	getConfigRPC      = "Admin.GetConfig"
-	writeTmpConfigRPC = "Admin.WriteTmpConfig"
-	commitConfigRPC   = "Admin.CommitConfig"
-)
+var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
 
-// localAdminClient - represents admin operation to be executed locally.
-type localAdminClient struct {
+// AdminRPCClient - admin RPC client talks to admin RPC server.
+type AdminRPCClient struct {
+	*RPCClient
 }
 
-// remoteAdminClient - represents admin operation to be executed
-// remotely, via RPC.
-type remoteAdminClient struct {
-	*AuthRPCClient
+// SignalService - calls SignalService RPC.
+func (rpcClient *AdminRPCClient) SignalService(signal serviceSignal) (err error) {
+	args := SignalServiceArgs{Sig: signal}
+	reply := VoidReply{}
+
+	return rpcClient.Call(adminServiceName+".SignalService", &args, &reply)
+}
+
+// ReInitFormat - re-initialize disk format, remotely.
+func (rpcClient *AdminRPCClient) ReInitFormat(dryRun bool) error {
+	args := ReInitFormatArgs{DryRun: dryRun}
+	reply := VoidReply{}
+
+	return rpcClient.Call(adminServiceName+".ReInitFormat", &args, &reply)
+}
+
+// ListLocks - Sends list locks command to remote server via RPC.
+func (rpcClient *AdminRPCClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
+	args := ListLocksQuery{
+		Bucket:   bucket,
+		Prefix:   prefix,
+		Duration: duration,
+	}
+	var reply []VolumeLockInfo
+
+	err := rpcClient.Call(adminServiceName+".ListLocks", &args, &reply)
+	return reply, err
+}
+
+// ServerInfo - returns the server info of the server to which the RPC call is made.
+func (rpcClient *AdminRPCClient) ServerInfo() (sid ServerInfoData, err error) {
+	err = rpcClient.Call(adminServiceName+".ServerInfo", &AuthArgs{}, &sid)
+	return sid, err
+}
+
+// GetConfig - returns config.json of the remote server.
+func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
+	args := AuthArgs{}
+	var reply []byte
+
+	err := rpcClient.Call(adminServiceName+".GetConfig", &args, &reply)
+	return reply, err
+}
+
+// WriteTmpConfig - writes config file content to a temporary file on a remote node.
+func (rpcClient *AdminRPCClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
+	args := WriteConfigArgs{
+		TmpFileName: tmpFileName,
+		Buf:         configBytes,
+	}
+	reply := VoidReply{}
+
+	err := rpcClient.Call(adminServiceName+".WriteTmpConfig", &args, &reply)
+	logger.LogIf(context.Background(), err)
+	return err
+}
+
+// CommitConfig - Move the new config in tmpFileName onto config.json on a remote node.
+func (rpcClient *AdminRPCClient) CommitConfig(tmpFileName string) error {
+	args := CommitConfigArgs{FileName: tmpFileName}
+	reply := VoidReply{}
+
+	err := rpcClient.Call(adminServiceName+".CommitConfig", &args, &reply)
+	logger.LogIf(context.Background(), err)
+	return err
+}
+
+// NewAdminRPCClient - returns new admin RPC client.
+func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
+	scheme := "http"
+	if globalIsSSL {
+		scheme = "https"
+	}
+
+	serviceURL := &xnet.URL{
+		Scheme: scheme,
+		Host:   host.String(),
+		Path:   adminServicePath,
+	}
+
+	var tlsConfig *tls.Config
+	if globalIsSSL {
+		tlsConfig = &tls.Config{
+			ServerName: host.Name,
+			RootCAs:    globalRootCAs,
+		}
+	}
+
+	rpcClient, err := NewRPCClient(
+		RPCClientArgs{
+			NewAuthTokenFunc: newAuthToken,
+			RPCVersion:       globalRPCAPIVersion,
+			ServiceName:      adminServiceName,
+			ServiceURL:       serviceURL,
+			TLSConfig:        tlsConfig,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &AdminRPCClient{rpcClient}, nil
 }
 
 // adminCmdRunner - abstracts local and remote execution of admin
@@ -59,182 +148,12 @@ type adminCmdRunner interface {
 	SignalService(s serviceSignal) error
 	ReInitFormat(dryRun bool) error
 	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
-	ServerInfoData() (ServerInfoData, error)
+	ServerInfo() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
 	WriteTmpConfig(tmpFileName string, configBytes []byte) error
 	CommitConfig(tmpFileName string) error
 }
 
-var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
-
-// SignalService - sends a restart or stop signal to the local server
-func (lc localAdminClient) SignalService(s serviceSignal) error {
-	switch s {
-	case serviceRestart, serviceStop:
-		globalServiceSignalCh <- s
-	default:
-		return errUnsupportedSignal
-	}
-	return nil
-}
-
-// ReInitFormat - re-initialize disk format.
-func (lc localAdminClient) ReInitFormat(dryRun bool) error {
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	_, err := objectAPI.HealFormat(context.Background(), dryRun)
-	return err
-}
-
-// ListLocks - Fetches lock information from local lock instrumentation.
-func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	return listLocksInfo(bucket, prefix, duration), nil
-}
-
-func (rc remoteAdminClient) SignalService(s serviceSignal) (err error) {
-	switch s {
-	case serviceRestart, serviceStop:
-		reply := AuthRPCReply{}
-		err = rc.Call(signalServiceRPC, &SignalServiceArgs{Sig: s},
-			&reply)
-	default:
-		err = errUnsupportedSignal
-	}
-	return err
-}
-
-// ReInitFormat - re-initialize disk format, remotely.
-func (rc remoteAdminClient) ReInitFormat(dryRun bool) error {
-	reply := AuthRPCReply{}
-	return rc.Call(reInitFormatRPC, &ReInitFormatArgs{
-		DryRun: dryRun,
-	}, &reply)
-}
-
-// ListLocks - Sends list locks command to remote server via RPC.
-func (rc remoteAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	listArgs := ListLocksQuery{
-		Bucket:   bucket,
-		Prefix:   prefix,
-		Duration: duration,
-	}
-	var reply ListLocksReply
-	if err := rc.Call(listLocksRPC, &listArgs, &reply); err != nil {
-		return nil, err
-	}
-	return reply.VolLocks, nil
-}
-
-// ServerInfoData - Returns the server info of this server.
-func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	if globalBootTime.IsZero() {
-		return sid, errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return sid, errServerNotInitialized
-	}
-	storage := objLayer.StorageInfo(context.Background())
-
-	return ServerInfoData{
-		StorageInfo: storage,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			SQSARN:   globalNotificationSys.GetARNList(),
-			Region:   globalServerConfig.GetRegion(),
-		},
-	}, nil
-}
-
-// ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	args := AuthRPCArgs{}
-	reply := ServerInfoDataReply{}
-	err := rc.Call(serverInfoDataRPC, &args, &reply)
-	if err != nil {
-		return sid, err
-	}
-
-	return reply.ServerInfoData, nil
-}
-
-// GetConfig - returns config.json of the local server.
-func (lc localAdminClient) GetConfig() ([]byte, error) {
-	if globalServerConfig == nil {
-		return nil, fmt.Errorf("config not present")
-	}
-
-	return json.Marshal(globalServerConfig)
-}
-
-// GetConfig - returns config.json of the remote server.
-func (rc remoteAdminClient) GetConfig() ([]byte, error) {
-	args := AuthRPCArgs{}
-	reply := ConfigReply{}
-	if err := rc.Call(getConfigRPC, &args, &reply); err != nil {
-		return nil, err
-	}
-	return reply.Config, nil
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// the local server.
-func (lc localAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	return writeTmpConfigCommon(tmpFileName, configBytes)
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// a remote node.
-func (rc remoteAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	wArgs := WriteConfigArgs{
-		TmpFileName: tmpFileName,
-		Buf:         configBytes,
-	}
-
-	err := rc.Call(writeTmpConfigRPC, &wArgs, &WriteConfigReply{})
-	if err != nil {
-		errorIf(err, "Failed to write temporary config file.")
-		return err
-	}
-
-	return nil
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a local node.
-func (lc localAdminClient) CommitConfig(tmpFileName string) error {
-	configFile := getConfigFile()
-	tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName)
-
-	err := os.Rename(tmpConfigFile, configFile)
-	errorIf(err, fmt.Sprintf("Failed to rename %s to %s", tmpConfigFile, configFile))
-	return err
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a remote node.
-func (rc remoteAdminClient) CommitConfig(tmpFileName string) error {
-	cArgs := CommitConfigArgs{
-		FileName: tmpFileName,
-	}
-	cReply := CommitConfigReply{}
-	err := rc.Call(commitConfigRPC, &cArgs, &cReply)
-	if err != nil {
-		errorIf(err, "Failed to rename config file.")
-		return err
-	}
-
-	return nil
-}
-
 // adminPeer - represents an entity that implements admin API RPCs.
 type adminPeer struct {
 	addr      string
@@ -247,34 +166,25 @@ type adminPeers []adminPeer
 
 // makeAdminPeers - helper function to construct a collection of adminPeer.
 func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
-	thisPeer := globalMinioAddr
-	if globalMinioHost == "" {
-		thisPeer = net.JoinHostPort("localhost", globalMinioPort)
+	localAddr := GetLocalPeer(endpoints)
+	if strings.HasPrefix(localAddr, "127.0.0.1:") {
+		// Use first IPv4 instead of loopback address.
+		localAddr = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
 	}
 	adminPeerList = append(adminPeerList, adminPeer{
-		thisPeer,
-		localAdminClient{},
-		true,
+		addr:      localAddr,
+		cmdRunner: localAdminClient{},
+		isLocal:   true,
 	})
 
-	hostSet := set.CreateStringSet(globalMinioAddr)
-	cred := globalServerConfig.GetCredential()
-	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
-	for _, host := range GetRemotePeers(endpoints) {
-		if hostSet.Contains(host) {
-			continue
-		}
-		hostSet.Add(host)
+	for _, hostStr := range GetRemotePeers(endpoints) {
+		host, err := xnet.ParseHost(hostStr)
+		logger.FatalIf(err, "Unable to parse Admin RPC Host", context.Background())
+		rpcClient, err := NewAdminRPCClient(host)
+		logger.FatalIf(err, "Unable to initialize Admin RPC Client", context.Background())
 		adminPeerList = append(adminPeerList, adminPeer{
-			addr: host,
-			cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
-				accessKey:       cred.AccessKey,
-				secretKey:       cred.SecretKey,
-				serverAddr:      host,
-				serviceEndpoint: serviceEndpoint,
-				secureConn:      globalIsSSL,
-				serviceName:     "Admin",
-			})},
+			addr:      hostStr,
+			cmdRunner: rpcClient,
 		})
 	}
 
@@ -420,7 +330,7 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
 		wg.Add(1)
 		go func(idx int, peer adminPeer) {
 			defer wg.Done()
-			serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
+			serverInfoData, rpcErr := peer.cmdRunner.ServerInfo()
 			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
 		}(i, peer)
 	}
@@ -436,7 +346,7 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
 	latestUptime := time.Duration(0)
 	for _, uptime := range uptimes {
 		if uptime.err != nil {
-			errorIf(uptime.err, "Unable to fetch uptime")
+			logger.LogIf(context.Background(), uptime.err)
 			continue
 		}
 
@@ -489,15 +399,17 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
 		// Unmarshal the received config files.
 		err := json.Unmarshal(configBytes, &serverConfigs[i])
 		if err != nil {
-			errorIf(err, "Failed to unmarshal serverConfig from ", peers[i].addr)
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peers[i].addr)
+			ctx := logger.SetReqInfo(context.Background(), reqInfo)
+			logger.LogIf(ctx, err)
 			return nil, err
 		}
 	}
 
 	configJSON, err := getValidServerConfig(serverConfigs, errs)
 	if err != nil {
-		errorIf(err, "Unable to find a valid server config")
-		return nil, errors.Trace(err)
+		logger.LogIf(context.Background(), err)
+		return nil, err
 	}
 
 	// Return the config.json that was present quorum or more

cmd/admin-rpc-client_test.go

@@ -1,260 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"encoding/json"
-	"reflect"
-	"testing"
-)
-
-var (
-	config1 = []byte(`{
-	"version": "13",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "us-east-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "debug"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	}
-}
-`)
-	// diff from config1 - amqp.Enable is True
-	config2 = []byte(`{
-	"version": "13",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "us-east-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "debug"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": true,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	}
-}
-`)
-)
-
-// TestGetValidServerConfig - test for getValidServerConfig.
-func TestGetValidServerConfig(t *testing.T) {
-	var c1, c2 serverConfig
-	err := json.Unmarshal(config1, &c1)
-	if err != nil {
-		t.Fatalf("json unmarshal of %s failed: %v", string(config1), err)
-	}
-
-	err = json.Unmarshal(config2, &c2)
-	if err != nil {
-		t.Fatalf("json unmarshal of %s failed: %v", string(config2), err)
-	}
-
-	// Valid config.
-	noErrs := []error{nil, nil, nil, nil}
-	serverConfigs := []serverConfig{c1, c2, c1, c1}
-	validConfig, err := getValidServerConfig(serverConfigs, noErrs)
-	if err != nil {
-		t.Errorf("Expected a valid config but received %v instead", err)
-	}
-
-	if !reflect.DeepEqual(validConfig, c1) {
-		t.Errorf("Expected valid config to be %v but received %v", config1, validConfig)
-	}
-
-	// Invalid config - no quorum.
-	serverConfigs = []serverConfig{c1, c2, c2, c1}
-	_, err = getValidServerConfig(serverConfigs, noErrs)
-	if err != errXLWriteQuorum {
-		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
-	}
-
-	// All errors
-	allErrs := []error{errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound}
-	serverConfigs = []serverConfig{{}, {}, {}, {}}
-	_, err = getValidServerConfig(serverConfigs, allErrs)
-	if err != errXLWriteQuorum {
-		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
-	}
-}

cmd/admin-rpc-server.go

@@ -18,211 +18,110 @@ package cmd
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
+	"path"
 	"time"
 
-	router "github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	xrpc "github.com/minio/minio/cmd/rpc"
 )
 
-const adminPath = "/admin"
+const adminServiceName = "Admin"
+const adminServiceSubPath = "/admin"
 
-// adminCmd - exports RPC methods for service status, stop and
-// restart commands.
-type adminCmd struct {
-	AuthRPCServer
+var adminServicePath = path.Join(minioReservedBucketPath, adminServiceSubPath)
+
+// adminRPCReceiver - Admin RPC receiver for admin RPC server.
+type adminRPCReceiver struct {
+	local *localAdminClient
 }
 
 // SignalServiceArgs - provides the signal argument to SignalService RPC
 type SignalServiceArgs struct {
-	AuthRPCArgs
+	AuthArgs
 	Sig serviceSignal
 }
 
+// SignalService - Send a restart or stop signal to the service
+func (receiver *adminRPCReceiver) SignalService(args *SignalServiceArgs, reply *VoidReply) error {
+	return receiver.local.SignalService(args.Sig)
+}
+
 // ListLocksQuery - wraps ListLocks API's query values to send over RPC.
 type ListLocksQuery struct {
-	AuthRPCArgs
+	AuthArgs
 	Bucket   string
 	Prefix   string
 	Duration time.Duration
 }
 
-// ListLocksReply - wraps ListLocks response over RPC.
-type ListLocksReply struct {
-	AuthRPCReply
-	VolLocks []VolumeLockInfo
+// ListLocks - lists locks held by requests handled by this server instance.
+func (receiver *adminRPCReceiver) ListLocks(args *ListLocksQuery, reply *[]VolumeLockInfo) (err error) {
+	*reply, err = receiver.local.ListLocks(args.Bucket, args.Prefix, args.Duration)
+	return err
 }
 
-// ServerInfoDataReply - wraps the server info response over RPC.
-type ServerInfoDataReply struct {
-	AuthRPCReply
-	ServerInfoData ServerInfoData
+// ServerInfo - returns the server info when object layer was initialized on this server.
+func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoData) (err error) {
+	*reply, err = receiver.local.ServerInfo()
+	return err
 }
 
-// ConfigReply - wraps the server config response over RPC.
-type ConfigReply struct {
-	AuthRPCReply
-	Config []byte // json-marshalled bytes of serverConfigV13
-}
-
-// SignalService - Send a restart or stop signal to the service
-func (s *adminCmd) SignalService(args *SignalServiceArgs, reply *AuthRPCReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	globalServiceSignalCh <- args.Sig
-	return nil
+// GetConfig - returns the config.json of this server.
+func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
+	*reply, err = receiver.local.GetConfig()
+	return err
 }
 
 // ReInitFormatArgs - provides dry-run information to re-initialize format.json
 type ReInitFormatArgs struct {
-	AuthRPCArgs
+	AuthArgs
 	DryRun bool
 }
 
 // ReInitFormat - re-init 'format.json'
-func (s *adminCmd) ReInitFormat(args *ReInitFormatArgs, reply *AuthRPCReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	_, err := objectAPI.HealFormat(context.Background(), args.DryRun)
-	return err
-}
-
-// ListLocks - lists locks held by requests handled by this server instance.
-func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error {
-	if err := query.IsAuthenticated(); err != nil {
-		return err
-	}
-	volLocks := listLocksInfo(query.Bucket, query.Prefix, query.Duration)
-	*reply = ListLocksReply{VolLocks: volLocks}
-	return nil
-}
-
-// ServerInfo - returns the server info when object layer was initialized on this server.
-func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	if globalBootTime.IsZero() {
-		return errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return errServerNotInitialized
-	}
-	storageInfo := objLayer.StorageInfo(context.Background())
-
-	reply.ServerInfoData = ServerInfoData{
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			Region:   globalServerConfig.GetRegion(),
-			SQSARN:   globalNotificationSys.GetARNList(),
-		},
-		StorageInfo: storageInfo,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-	}
-
-	return nil
-}
-
-// GetConfig - returns the config.json of this server.
-func (s *adminCmd) GetConfig(args *AuthRPCArgs, reply *ConfigReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	if globalServerConfig == nil {
-		return fmt.Errorf("config not present")
-	}
-
-	jsonBytes, err := json.Marshal(globalServerConfig)
-	if err != nil {
-		return err
-	}
-
-	reply.Config = jsonBytes
-	return nil
+func (receiver *adminRPCReceiver) ReInitFormat(args *ReInitFormatArgs, reply *VoidReply) error {
+	return receiver.local.ReInitFormat(args.DryRun)
 }
 
 // WriteConfigArgs - wraps the bytes to be written and temporary file name.
 type WriteConfigArgs struct {
-	AuthRPCArgs
+	AuthArgs
 	TmpFileName string
 	Buf         []byte
 }
 
-// WriteConfigReply - wraps the result of a writing config into a temporary file.
-// the remote node.
-type WriteConfigReply struct {
-	AuthRPCReply
-}
-
-func writeTmpConfigCommon(tmpFileName string, configBytes []byte) error {
-	tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName)
-	err := ioutil.WriteFile(tmpConfigFile, configBytes, 0666)
-	errorIf(err, fmt.Sprintf("Failed to write to temporary config file %s", tmpConfigFile))
-	return err
-}
-
 // WriteTmpConfig - writes the supplied config contents onto the
 // supplied temporary file.
-func (s *adminCmd) WriteTmpConfig(wArgs *WriteConfigArgs, wReply *WriteConfigReply) error {
-	if err := wArgs.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	return writeTmpConfigCommon(wArgs.TmpFileName, wArgs.Buf)
+func (receiver *adminRPCReceiver) WriteTmpConfig(args *WriteConfigArgs, reply *VoidReply) error {
+	return receiver.local.WriteTmpConfig(args.TmpFileName, args.Buf)
 }
 
 // CommitConfigArgs - wraps the config file name that needs to be
 // committed into config.json on this node.
 type CommitConfigArgs struct {
-	AuthRPCArgs
+	AuthArgs
 	FileName string
 }
 
-// CommitConfigReply - represents response to commit of config file on
-// this node.
-type CommitConfigReply struct {
-	AuthRPCReply
-}
-
 // CommitConfig - Renames the temporary file into config.json on this node.
-func (s *adminCmd) CommitConfig(cArgs *CommitConfigArgs, cReply *CommitConfigReply) error {
-	configFile := getConfigFile()
-	tmpConfigFile := filepath.Join(getConfigDir(), cArgs.FileName)
-
-	err := os.Rename(tmpConfigFile, configFile)
-	errorIf(err, fmt.Sprintf("Failed to rename %s to %s", tmpConfigFile, configFile))
-	return err
+func (receiver *adminRPCReceiver) CommitConfig(args *CommitConfigArgs, reply *VoidReply) error {
+	return receiver.local.CommitConfig(args.FileName)
 }
 
-// registerAdminRPCRouter - registers RPC methods for service status,
-// stop and restart commands.
-func registerAdminRPCRouter(mux *router.Router) error {
-	adminRPCHandler := &adminCmd{}
-	adminRPCServer := newRPCServer()
-	err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
-	if err != nil {
-		return errors.Trace(err)
+// NewAdminRPCServer - returns new admin RPC server.
+func NewAdminRPCServer() (*xrpc.Server, error) {
+	rpcServer := xrpc.NewServer()
+	if err := rpcServer.RegisterName(adminServiceName, &adminRPCReceiver{&localAdminClient{}}); err != nil {
+		return nil, err
 	}
-	adminRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
-	adminRouter.Path(adminPath).Handler(adminRPCServer)
-	return nil
+	return rpcServer, nil
+}
+
+// registerAdminRPCRouter - creates and registers Admin RPC server and its router.
+func registerAdminRPCRouter(router *mux.Router) {
+	rpcServer, err := NewAdminRPCServer()
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
+	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
+	subrouter.Path(adminServiceSubPath).Handler(rpcServer)
 }

cmd/admin-rpc-server_test.go

@@ -1,262 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"encoding/json"
-	"os"
-	"testing"
-)
-
-func testAdminCmd(cmd cmdType, t *testing.T) {
-	// reset globals. this is to make sure that the tests are not
-	// affected by modified globals.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatalf("Failed to create test config - %v", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	adminServer := adminCmd{}
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	err = adminServer.Login(&args, &LoginRPCReply{})
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	go func() {
-		// A test signal receiver
-		<-globalServiceSignalCh
-	}()
-
-	sa := SignalServiceArgs{
-		AuthRPCArgs: AuthRPCArgs{AuthToken: token, Version: globalRPCAPIVersion},
-		Sig:         cmd.toServiceSignal(),
-	}
-
-	genReply := AuthRPCReply{}
-	switch cmd {
-	case restartCmd, stopCmd:
-		if err = adminServer.SignalService(&sa, &genReply); err != nil {
-			t.Errorf("restartCmd/stopCmd: Expected: <nil>, got: %v",
-				err)
-		}
-	default:
-		err = adminServer.SignalService(&sa, &genReply)
-		if err != nil && err.Error() != errUnsupportedSignal.Error() {
-			t.Errorf("invalidSignal %s: unexpected error got: %v",
-				cmd, err)
-		}
-	}
-}
-
-// TestAdminRestart - test for Admin.Restart RPC service.
-func TestAdminRestart(t *testing.T) {
-	testAdminCmd(restartCmd, t)
-}
-
-// TestAdminStop - test for Admin.Stop RPC service.
-func TestAdminStop(t *testing.T) {
-	testAdminCmd(stopCmd, t)
-}
-
-// TestAdminStatus - test for Admin.Status RPC service (error case)
-func TestAdminStatus(t *testing.T) {
-	testAdminCmd(statusCmd, t)
-}
-
-// TestReInitFormat - test for Admin.ReInitFormat RPC service.
-func TestReInitFormat(t *testing.T) {
-	// Reset global variables to start afresh.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig("us-east-1")
-	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	// Initializing objectLayer for HealFormatHandler.
-	_, xlDirs, xlErr := initTestXLObjLayer()
-	if xlErr != nil {
-		t.Fatalf("failed to initialize XL based object layer - %v.", xlErr)
-	}
-	defer removeRoots(xlDirs)
-
-	// Set globalEndpoints for a single node XL setup.
-	globalEndpoints = mustGetNewEndpointList(xlDirs...)
-
-	// Setup admin rpc server for an XL backend.
-	globalIsXL = true
-	adminServer := adminCmd{}
-
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	err = adminServer.Login(&args, &LoginRPCReply{})
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	authArgs := AuthRPCArgs{
-		AuthToken: token,
-		Version:   globalRPCAPIVersion,
-	}
-	authReply := AuthRPCReply{}
-
-	err = adminServer.ReInitFormat(&ReInitFormatArgs{
-		AuthRPCArgs: authArgs,
-		DryRun:      false,
-	}, &authReply)
-	if err != nil {
-		t.Errorf("Expected to pass, but failed with %v", err)
-	}
-}
-
-// TestGetConfig - Test for GetConfig admin RPC.
-func TestGetConfig(t *testing.T) {
-	// Reset global variables to start afresh.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig("us-east-1")
-	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	adminServer := adminCmd{}
-	creds := globalServerConfig.GetCredential()
-
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	reply := LoginRPCReply{}
-	err = adminServer.Login(&args, &reply)
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	authArgs := AuthRPCArgs{
-		AuthToken: token,
-		Version:   globalRPCAPIVersion,
-	}
-
-	configReply := ConfigReply{}
-
-	err = adminServer.GetConfig(&authArgs, &configReply)
-	if err != nil {
-		t.Errorf("Expected GetConfig to pass but failed with %v", err)
-	}
-
-	var config serverConfigV13
-	err = json.Unmarshal(configReply.Config, &config)
-	if err != nil {
-		t.Errorf("Expected json unmarshal to pass but failed with %v", err)
-	}
-}
-
-// TestWriteAndCommitConfig - test for WriteTmpConfig and CommitConfig
-// RPC handler.
-func TestWriteAndCommitConfig(t *testing.T) {
-	// Reset global variables to start afresh.
-	resetTestGlobals()
-
-	rootPath, err := newTestConfig("us-east-1")
-	if err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	adminServer := adminCmd{}
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	args := LoginRPCArgs{
-		AuthToken:   token,
-		Version:     globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-	reply := LoginRPCReply{}
-	err = adminServer.Login(&args, &reply)
-	if err != nil {
-		t.Fatalf("Failed to login to admin server - %v", err)
-	}
-
-	// Write temporary config.
-	buf := []byte("hello")
-	tmpFileName := mustGetUUID()
-	wArgs := WriteConfigArgs{
-		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: token,
-			Version:   globalRPCAPIVersion,
-		},
-		TmpFileName: tmpFileName,
-		Buf:         buf,
-	}
-
-	err = adminServer.WriteTmpConfig(&wArgs, &WriteConfigReply{})
-	if err != nil {
-		t.Fatalf("Failed to write temporary config %v", err)
-	}
-
-	if err != nil {
-		t.Errorf("Expected to succeed but failed %v", err)
-	}
-
-	cArgs := CommitConfigArgs{
-		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: token,
-			Version:   globalRPCAPIVersion,
-		},
-		FileName: tmpFileName,
-	}
-	cReply := CommitConfigReply{}
-
-	err = adminServer.CommitConfig(&cArgs, &cReply)
-	if err != nil {
-		t.Fatalf("Failed to commit config file %v", err)
-	}
-}

cmd/admin-rpc_test.go

@@ -0,0 +1,611 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// localAdminClient and AdminRPCClient are adminCmdRunner interface compatible,
+// hence below test functions are available for both clients.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// Admin RPC server, adminRPCReceiver and AdminRPCClient are
+// inter-dependent, below test functions are sufficient to test all of them.
+//
+///////////////////////////////////////////////////////////////////////////////
+
+func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
+	tmpGlobalServiceSignalCh := globalServiceSignalCh
+	globalServiceSignalCh = make(chan serviceSignal, 10)
+	defer func() {
+		globalServiceSignalCh = tmpGlobalServiceSignalCh
+	}()
+
+	testCases := []struct {
+		signal    serviceSignal
+		expectErr bool
+	}{
+		{serviceRestart, false},
+		{serviceStop, false},
+		{serviceStatus, true},
+		{serviceSignal(100), true},
+	}
+
+	for i, testCase := range testCases {
+		err := client.SignalService(testCase.signal)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerReInitFormat(t *testing.T, client adminCmdRunner) {
+	tmpGlobalObjectAPI := globalObjectAPI
+	defer func() {
+		globalObjectAPI = tmpGlobalObjectAPI
+	}()
+
+	testCases := []struct {
+		objectAPI ObjectLayer
+		dryRun    bool
+		expectErr bool
+	}{
+		{&DummyObjectLayer{}, true, false},
+		{&DummyObjectLayer{}, false, false},
+		{nil, true, true},
+		{nil, false, true},
+	}
+
+	for i, testCase := range testCases {
+		globalObjectAPI = testCase.objectAPI
+		err := client.ReInitFormat(testCase.dryRun)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerListLocks(t *testing.T, client adminCmdRunner) {
+	tmpGlobalObjectAPI := globalObjectAPI
+	defer func() {
+		globalObjectAPI = tmpGlobalObjectAPI
+	}()
+
+	testCases := []struct {
+		objectAPI ObjectLayer
+		expectErr bool
+	}{
+		{&DummyObjectLayer{}, false},
+		{nil, true},
+	}
+
+	for i, testCase := range testCases {
+		globalObjectAPI = testCase.objectAPI
+		_, err := client.ListLocks("", "", time.Duration(0))
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
+	tmpGlobalBootTime := globalBootTime
+	tmpGlobalObjectAPI := globalObjectAPI
+	tmpGlobalConnStats := globalConnStats
+	tmpGlobalHTTPStats := globalHTTPStats
+	tmpGlobalNotificationSys := globalNotificationSys
+	defer func() {
+		globalBootTime = tmpGlobalBootTime
+		globalObjectAPI = tmpGlobalObjectAPI
+		globalConnStats = tmpGlobalConnStats
+		globalHTTPStats = tmpGlobalHTTPStats
+		globalNotificationSys = tmpGlobalNotificationSys
+	}()
+
+	endpoints := new(EndpointList)
+
+	notificationSys := NewNotificationSys(globalServerConfig, *endpoints)
+
+	testCases := []struct {
+		bootTime        time.Time
+		objectAPI       ObjectLayer
+		connStats       *ConnStats
+		httpStats       *HTTPStats
+		notificationSys *NotificationSys
+		expectErr       bool
+	}{
+		{UTCNow(), &DummyObjectLayer{}, newConnStats(), newHTTPStats(), notificationSys, false},
+		{time.Time{}, nil, nil, nil, nil, true},
+		{UTCNow(), nil, nil, nil, nil, true},
+	}
+
+	for i, testCase := range testCases {
+		globalBootTime = testCase.bootTime
+		globalObjectAPI = testCase.objectAPI
+		globalConnStats = testCase.connStats
+		globalHTTPStats = testCase.httpStats
+		globalNotificationSys = testCase.notificationSys
+		_, err := client.ServerInfo()
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerGetConfig(t *testing.T, client adminCmdRunner) {
+	tmpGlobalServerConfig := globalServerConfig
+	defer func() {
+		globalServerConfig = tmpGlobalServerConfig
+	}()
+
+	config := newServerConfig()
+
+	testCases := []struct {
+		config    *serverConfig
+		expectErr bool
+	}{
+		{globalServerConfig, false},
+		{config, false},
+	}
+
+	for i, testCase := range testCases {
+		globalServerConfig = testCase.config
+		_, err := client.GetConfig()
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerWriteTmpConfig(t *testing.T, client adminCmdRunner) {
+	tmpConfigDir := configDir
+	defer func() {
+		configDir = tmpConfigDir
+	}()
+
+	tempDir, err := ioutil.TempDir("", ".AdminCmdRunnerWriteTmpConfig.")
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+	configDir = &ConfigDir{dir: tempDir}
+
+	testCases := []struct {
+		tmpFilename string
+		configBytes []byte
+		expectErr   bool
+	}{
+		{"config1.json", []byte(`{"version":"23","region":"us-west-1a"}`), false},
+		// Overwrite test.
+		{"config1.json", []byte(`{"version":"23","region":"us-west-1a","browser":"on"}`), false},
+		{"config2.json", []byte{}, false},
+		{"config3.json", nil, false},
+	}
+
+	for i, testCase := range testCases {
+		err := client.WriteTmpConfig(testCase.tmpFilename, testCase.configBytes)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func testAdminCmdRunnerCommitConfig(t *testing.T, client adminCmdRunner) {
+	tmpConfigDir := configDir
+	defer func() {
+		configDir = tmpConfigDir
+	}()
+
+	tempDir, err := ioutil.TempDir("", ".AdminCmdRunnerCommitConfig.")
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+	configDir = &ConfigDir{dir: tempDir}
+	err = ioutil.WriteFile(filepath.Join(tempDir, "config.json"), []byte{}, os.ModePerm)
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	err = client.WriteTmpConfig("config1.json", []byte(`{"version":"23","region":"us-west-1a"}`))
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	testCases := []struct {
+		tmpFilename string
+		expectErr   bool
+	}{
+		{"config1.json", false},
+		{"config2.json", true},
+	}
+
+	for i, testCase := range testCases {
+		err := client.CommitConfig(testCase.tmpFilename)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+		}
+	}
+}
+
+func newAdminRPCHTTPServerClient(t *testing.T) (*httptest.Server, *AdminRPCClient, *serverConfig) {
+	rpcServer, err := NewAdminRPCServer()
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		rpcServer.ServeHTTP(w, r)
+	}))
+
+	url, err := xnet.ParseURL(httpServer.URL)
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	host, err := xnet.ParseHost(url.Host)
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	prevGlobalServerConfig := globalServerConfig
+	globalServerConfig = newServerConfig()
+
+	rpcClient, err := NewAdminRPCClient(host)
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	return httpServer, rpcClient, prevGlobalServerConfig
+}
+
+func TestAdminRPCClientSignalService(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerSignalService(t, rpcClient)
+}
+
+func TestAdminRPCClientReInitFormat(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerReInitFormat(t, rpcClient)
+}
+
+func TestAdminRPCClientListLocks(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerListLocks(t, rpcClient)
+}
+
+func TestAdminRPCClientServerInfo(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerServerInfo(t, rpcClient)
+}
+
+func TestAdminRPCClientGetConfig(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerGetConfig(t, rpcClient)
+}
+
+func TestAdminRPCClientWriteTmpConfig(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerWriteTmpConfig(t, rpcClient)
+}
+
+func TestAdminRPCClientCommitConfig(t *testing.T) {
+	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
+	defer httpServer.Close()
+	defer func() {
+		globalServerConfig = prevGlobalServerConfig
+	}()
+
+	testAdminCmdRunnerCommitConfig(t, rpcClient)
+}
+
+var (
+	config1 = []byte(`{
+	"version": "13",
+	"credential": {
+		"accessKey": "minio",
+		"secretKey": "minio123"
+	},
+	"region": "us-east-1",
+	"logger": {
+		"console": {
+			"enable": true,
+			"level": "debug"
+		},
+		"file": {
+			"enable": false,
+			"fileName": "",
+			"level": ""
+		}
+	},
+	"notify": {
+		"amqp": {
+			"1": {
+				"enable": false,
+				"url": "",
+				"exchange": "",
+				"routingKey": "",
+				"exchangeType": "",
+				"mandatory": false,
+				"immediate": false,
+				"durable": false,
+				"internal": false,
+				"noWait": false,
+				"autoDeleted": false
+			}
+		},
+		"nats": {
+			"1": {
+				"enable": false,
+				"address": "",
+				"subject": "",
+				"username": "",
+				"password": "",
+				"token": "",
+				"secure": false,
+				"pingInterval": 0,
+				"streaming": {
+					"enable": false,
+					"clusterID": "",
+					"clientID": "",
+					"async": false,
+					"maxPubAcksInflight": 0
+				}
+			}
+		},
+		"elasticsearch": {
+			"1": {
+				"enable": false,
+				"url": "",
+				"index": ""
+			}
+		},
+		"redis": {
+			"1": {
+				"enable": false,
+				"address": "",
+				"password": "",
+				"key": ""
+			}
+		},
+		"postgresql": {
+			"1": {
+				"enable": false,
+				"connectionString": "",
+				"table": "",
+				"host": "",
+				"port": "",
+				"user": "",
+				"password": "",
+				"database": ""
+			}
+		},
+		"kafka": {
+			"1": {
+				"enable": false,
+				"brokers": null,
+				"topic": ""
+			}
+		},
+		"webhook": {
+			"1": {
+				"enable": false,
+				"endpoint": ""
+			}
+		}
+	}
+}
+`)
+	// diff from config1 - amqp.Enable is True
+	config2 = []byte(`{
+	"version": "13",
+	"credential": {
+		"accessKey": "minio",
+		"secretKey": "minio123"
+	},
+	"region": "us-east-1",
+	"logger": {
+		"console": {
+			"enable": true,
+			"level": "debug"
+		},
+		"file": {
+			"enable": false,
+			"fileName": "",
+			"level": ""
+		}
+	},
+	"notify": {
+		"amqp": {
+			"1": {
+				"enable": true,
+				"url": "",
+				"exchange": "",
+				"routingKey": "",
+				"exchangeType": "",
+				"mandatory": false,
+				"immediate": false,
+				"durable": false,
+				"internal": false,
+				"noWait": false,
+				"autoDeleted": false
+			}
+		},
+		"nats": {
+			"1": {
+				"enable": false,
+				"address": "",
+				"subject": "",
+				"username": "",
+				"password": "",
+				"token": "",
+				"secure": false,
+				"pingInterval": 0,
+				"streaming": {
+					"enable": false,
+					"clusterID": "",
+					"clientID": "",
+					"async": false,
+					"maxPubAcksInflight": 0
+				}
+			}
+		},
+		"elasticsearch": {
+			"1": {
+				"enable": false,
+				"url": "",
+				"index": ""
+			}
+		},
+		"redis": {
+			"1": {
+				"enable": false,
+				"address": "",
+				"password": "",
+				"key": ""
+			}
+		},
+		"postgresql": {
+			"1": {
+				"enable": false,
+				"connectionString": "",
+				"table": "",
+				"host": "",
+				"port": "",
+				"user": "",
+				"password": "",
+				"database": ""
+			}
+		},
+		"kafka": {
+			"1": {
+				"enable": false,
+				"brokers": null,
+				"topic": ""
+			}
+		},
+		"webhook": {
+			"1": {
+				"enable": false,
+				"endpoint": ""
+			}
+		}
+	}
+}
+`)
+)
+
+// TestGetValidServerConfig - test for getValidServerConfig.
+func TestGetValidServerConfig(t *testing.T) {
+	var c1, c2 serverConfig
+	err := json.Unmarshal(config1, &c1)
+	if err != nil {
+		t.Fatalf("json unmarshal of %s failed: %v", string(config1), err)
+	}
+
+	err = json.Unmarshal(config2, &c2)
+	if err != nil {
+		t.Fatalf("json unmarshal of %s failed: %v", string(config2), err)
+	}
+
+	// Valid config.
+	noErrs := []error{nil, nil, nil, nil}
+	serverConfigs := []serverConfig{c1, c2, c1, c1}
+	validConfig, err := getValidServerConfig(serverConfigs, noErrs)
+	if err != nil {
+		t.Errorf("Expected a valid config but received %v instead", err)
+	}
+
+	if !reflect.DeepEqual(validConfig, c1) {
+		t.Errorf("Expected valid config to be %v but received %v", config1, validConfig)
+	}
+
+	// Invalid config - no quorum.
+	serverConfigs = []serverConfig{c1, c2, c2, c1}
+	_, err = getValidServerConfig(serverConfigs, noErrs)
+	if err != errXLWriteQuorum {
+		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
+	}
+
+	// All errors
+	allErrs := []error{errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound}
+	serverConfigs = []serverConfig{{}, {}, {}, {}}
+	_, err = getValidServerConfig(serverConfigs, allErrs)
+	if err != errXLWriteQuorum {
+		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
+	}
+}

cmd/api-errors.go

@@ -17,12 +17,13 @@
 package cmd
 
 import (
+	"context"
 	"encoding/xml"
 	"fmt"
 	"net/http"
 
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 )
@@ -138,6 +139,7 @@ const (
 	ErrMissingSSECustomerKey
 	ErrMissingSSECustomerKeyMD5
 	ErrSSECustomerKeyMD5Mismatch
+	ErrInvalidSSECustomerParameters
 
 	// Bucket notification related errors.
 	ErrEventNotification
@@ -191,6 +193,7 @@ const (
 	ErrHealMissingBucket
 	ErrHealAlreadyRunning
 	ErrHealOverlappingPaths
+	ErrIncorrectContinuationToken
 )
 
 // error code to APIError structure, these fields carry respective
@@ -629,7 +632,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInsecureSSECustomerRequest: {
 		Code:           "InvalidRequest",
-		Description:    errInsecureSSERequest.Error(),
+		Description:    "Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrSSEMultipartEncrypted: {
@@ -639,7 +642,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrSSEEncryptedObject: {
 		Code:           "InvalidRequest",
-		Description:    errEncryptedObject.Error(),
+		Description:    "The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidEncryptionParameters: {
@@ -649,27 +652,32 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInvalidSSECustomerAlgorithm: {
 		Code:           "InvalidArgument",
-		Description:    errInvalidSSEAlgorithm.Error(),
+		Description:    "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidSSECustomerKey: {
 		Code:           "InvalidArgument",
-		Description:    errInvalidSSEKey.Error(),
+		Description:    "The secret key was invalid for the specified algorithm.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMissingSSECustomerKey: {
 		Code:           "InvalidArgument",
-		Description:    errMissingSSEKey.Error(),
+		Description:    "Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMissingSSECustomerKeyMD5: {
 		Code:           "InvalidArgument",
-		Description:    errMissingSSEKeyMD5.Error(),
+		Description:    "Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrSSECustomerKeyMD5Mismatch: {
 		Code:           "InvalidArgument",
-		Description:    errSSEKeyMD5Mismatch.Error(),
+		Description:    "The calculated MD5 hash of the key did not match the hash that was provided.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidSSECustomerParameters: {
+		Code:           "InvalidArgument",
+		Description:    "The provided encryption parameters did not match the ones used originally.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 
@@ -684,7 +692,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
 		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
-		HTTPStatusCode: http.StatusInternalServerError,
+		HTTPStatusCode: http.StatusInsufficientStorage,
 	},
 	ErrRequestBodyParse: {
 		Code:           "XMinioRequestBodyParse",
@@ -793,7 +801,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	// Generic Invalid-Request error. Should be used for response errors only for unlikely
-	// corner case errors for which introducing new APIErrorCode is not worth it. errorIf()
+	// corner case errors for which introducing new APIErrorCode is not worth it. LogIf()
 	// should be used to log the error at the source of the error for debugging purposes.
 	ErrInvalidRequest: {
 		Code:           "InvalidRequest",
@@ -835,7 +843,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Object storage backend is unreachable",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
-
+	ErrIncorrectContinuationToken: {
+		Code:           "InvalidArgument",
+		Description:    "The continuation token provided is incorrect",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 
@@ -847,7 +859,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
-	err = errors.Cause(err)
 	// Verify if the underlying error is signature mismatch.
 	switch err {
 	case errSignatureMismatch:
@@ -860,6 +871,29 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
+	// SSE errors
+	case errInsecureSSERequest:
+		apiErr = ErrInsecureSSECustomerRequest
+	case errInvalidSSEAlgorithm:
+		apiErr = ErrInvalidSSECustomerAlgorithm
+	case errInvalidSSEKey:
+		apiErr = ErrInvalidSSECustomerKey
+	case errMissingSSEKey:
+		apiErr = ErrMissingSSECustomerKey
+	case errMissingSSEKeyMD5:
+		apiErr = ErrMissingSSECustomerKeyMD5
+	case errSSEKeyMD5Mismatch:
+		apiErr = ErrSSECustomerKeyMD5Mismatch
+	case errObjectTampered:
+		apiErr = ErrObjectTampered
+	case errEncryptedObject:
+		apiErr = ErrSSEEncryptedObject
+	case errInvalidSSEParameters:
+		apiErr = ErrInvalidSSECustomerParameters
+	case errSSEKeyMismatch:
+		apiErr = ErrAccessDenied // no access without correct key
+	case context.Canceled, context.DeadlineExceeded:
+		apiErr = ErrOperationTimedOut
 	}
 
 	if apiErr != ErrNone {
@@ -867,25 +901,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return apiErr
 	}
 
-	switch err { // SSE errors
-	case errInsecureSSERequest:
-		return ErrInsecureSSECustomerRequest
-	case errInvalidSSEAlgorithm:
-		return ErrInvalidSSECustomerAlgorithm
-	case errInvalidSSEKey:
-		return ErrInvalidSSECustomerKey
-	case errMissingSSEKey:
-		return ErrMissingSSECustomerKey
-	case errMissingSSEKeyMD5:
-		return ErrMissingSSECustomerKeyMD5
-	case errSSEKeyMD5Mismatch:
-		return ErrSSECustomerKeyMD5Mismatch
-	case errObjectTampered:
-		return ErrObjectTampered
-	case errEncryptedObject:
-		return ErrSSEEncryptedObject
-	case errSSEKeyMismatch:
-		return ErrAccessDenied // no access without correct key
+	// etcd specific errors, a key is always a bucket for us return
+	// ErrNoSuchBucket in such a case.
+	if err == dns.ErrNoEntriesFound {
+		return ErrNoSuchBucket
 	}
 
 	switch err.(type) {
@@ -947,8 +966,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooSmall
 	case NotImplemented:
 		apiErr = ErrNotImplemented
-	case PolicyNotFound:
-		apiErr = ErrNoSuchBucketPolicy
 	case PartTooBig:
 		apiErr = ErrEntityTooLarge
 	case UnsupportedMetadata:

cmd/api-resources.go

@@ -36,7 +36,17 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
 }
 
 // Parse bucket url queries for ListObjects V2.
-func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string) {
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	// The continuation-token cannot be empty.
+	if val, ok := values["continuation-token"]; ok {
+		if len(val[0]) == 0 {
+			errCode = ErrIncorrectContinuationToken
+			return
+		}
+	}
+
 	prefix = values.Get("prefix")
 	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")
@@ -78,13 +88,3 @@ func getObjectResources(values url.Values) (uploadID string, partNumberMarker, m
 	encodingType = values.Get("encoding-type")
 	return
 }
-
-// Validates filter values
-func validateFilterValues(values []string) (err APIErrorCode) {
-	for _, value := range values {
-		if !IsValidObjectPrefix(value) {
-			return ErrFilterValueInvalid
-		}
-	}
-	return ErrNone
-}

cmd/api-resources_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"net/url"
-	"strings"
 	"testing"
 )
 
@@ -30,6 +29,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		fetchOwner                           bool
 		maxKeys                              int
 		encodingType                         string
+		errCode                              APIErrorCode
 	}{
 		{
 			values: url.Values{
@@ -48,6 +48,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      100,
 			encodingType: "gzip",
+			errCode:      ErrNone,
 		},
 		{
 			values: url.Values{
@@ -65,11 +66,34 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      1000,
 			encodingType: "gzip",
+			errCode:      ErrNone,
+		},
+		{
+			values: url.Values{
+				"prefix":             []string{"photos/"},
+				"continuation-token": []string{""},
+				"start-after":        []string{"start-after"},
+				"delimiter":          []string{"/"},
+				"fetch-owner":        []string{"true"},
+				"encoding-type":      []string{"gzip"},
+			},
+			prefix:       "",
+			token:        "",
+			startAfter:   "",
+			delimiter:    "",
+			fetchOwner:   false,
+			maxKeys:      0,
+			encodingType: "",
+			errCode:      ErrIncorrectContinuationToken,
 		},
 	}
 
 	for i, testCase := range testCases {
-		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType := getListObjectsV2Args(testCase.values)
+		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(testCase.values)
+
+		if errCode != testCase.errCode {
+			t.Errorf("Test %d: Expected error code:%d, got %d", i+1, testCase.errCode, errCode)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}
@@ -189,38 +213,3 @@ func TestGetObjectsResources(t *testing.T) {
 		}
 	}
 }
-
-// Validates if filter values are correct
-func TestValidateFilterValues(t *testing.T) {
-	testCases := []struct {
-		values        []string
-		expectedError APIErrorCode
-	}{
-		{
-			values:        []string{""},
-			expectedError: ErrNone,
-		},
-		{
-			values:        []string{"", "prefix"},
-			expectedError: ErrNone,
-		},
-		{
-			values:        []string{strings.Repeat("a", 1025)},
-			expectedError: ErrFilterValueInvalid,
-		},
-		{
-			values:        []string{"a\\b"},
-			expectedError: ErrFilterValueInvalid,
-		},
-		{
-			values:        []string{string([]byte{0xff, 0xfe, 0xfd})},
-			expectedError: ErrFilterValueInvalid,
-		},
-	}
-
-	for i, testCase := range testCases {
-		if actualError := validateFilterValues(testCase.values); actualError != testCase.expectedError {
-			t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.expectedError, actualError)
-		}
-	}
-}

cmd/api-response.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/pkg/handlers"
@@ -293,8 +294,10 @@ func getObjectLocation(r *http.Request, domain, bucket, object string) string {
 	}
 	// If domain is set then we need to use bucket DNS style.
 	if domain != "" {
-		u.Host = bucket + "." + domain
-		u.Path = path.Join(slashSeparator, object)
+		if strings.Contains(r.Host, domain) {
+			u.Host = bucket + "." + r.Host
+			u.Path = path.Join(slashSeparator, object)
+		}
 	}
 	return u.String()
 }
@@ -339,7 +342,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
-		content.StorageClass = globalMinioDefaultStorageClass
+		content.StorageClass = object.StorageClass
 		content.Owner = owner
 		contents = append(contents, content)
 	}
@@ -385,7 +388,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
-		content.StorageClass = globalMinioDefaultStorageClass
+		content.StorageClass = object.StorageClass
 		content.Owner = owner
 		contents = append(contents, content)
 	}

cmd/api-router.go

@@ -19,7 +19,8 @@ package cmd
 import (
 	"net/http"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // objectAPIHandler implements and provides http handlers for S3 API.
@@ -29,13 +30,13 @@ type objectAPIHandlers struct {
 }
 
 // registerAPIRouter - registers S3 compatible APIs.
-func registerAPIRouter(mux *router.Router) {
+func registerAPIRouter(router *mux.Router) {
 	var err error
 	var cacheConfig = globalServerConfig.GetCacheConfig()
 	if len(cacheConfig.Drives) > 0 {
 		// initialize the new disk cache objects.
 		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
-		fatalIf(err, "Unable to initialize disk caching")
+		logger.FatalIf(err, "Unable to initialize disk caching")
 	}
 
 	// Initialize API.
@@ -45,8 +46,8 @@ func registerAPIRouter(mux *router.Router) {
 	}
 
 	// API Router
-	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
-	var routers []*router.Router
+	apiRouter := router.PathPrefix("/").Subrouter()
+	var routers []*mux.Router
 	if globalDomainName != "" {
 		routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
 	}
@@ -68,6 +69,8 @@ func registerAPIRouter(mux *router.Router) {
 		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
 		// AbortMultipartUpload
 		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// GetObjectACL - this is a dummy call.
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
 		// GetObject
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
 		// CopyObject
@@ -82,6 +85,10 @@ func registerAPIRouter(mux *router.Router) {
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
 		// GetBucketPolicy
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+
+		// GetBucketACL -- this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
+
 		// GetBucketNotification
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
 		// ListenBucketNotification

cmd/auth-handler.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015-2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,14 +18,18 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"strings"
 
-	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // Verify if request has JWT.
@@ -110,37 +114,77 @@ func getRequestAuthType(r *http.Request) authType {
 // It does not accept presigned or JWT or anonymous requests.
 func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
 	s3Err := ErrAccessDenied
-	if getRequestAuthType(r) == authTypeSigned { // we only support V4 (no presign)
+	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok && getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) { // we only support V4 (no presign) with auth. body
 		s3Err = isReqAuthenticated(r, region)
 	}
 	if s3Err != ErrNone {
-		errorIf(errors.New(getAPIError(s3Err).Description), "%s", dumpRequest(r))
+		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description))
 	}
 	return s3Err
 }
 
-func checkRequestAuthType(r *http.Request, bucket, policyAction, region string) APIErrorCode {
-	reqAuthType := getRequestAuthType(r)
+func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) APIErrorCode {
+	isOwner := true
+	accountName := globalServerConfig.GetCredential().AccessKey
 
-	switch reqAuthType {
+	switch getRequestAuthType(r) {
+	case authTypeUnknown:
+		return ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
-		// Signature V2 validation.
-		return isReqAuthenticatedV2(r)
-	case authTypeSigned, authTypePresigned:
-		return isReqAuthenticated(r, region)
-	}
-
-	if reqAuthType == authTypeAnonymous && policyAction != "" {
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
-		if err != nil {
-			return ErrInternalError
+		if errorCode := isReqAuthenticatedV2(r); errorCode != ErrNone {
+			return errorCode
 		}
-		return enforceBucketPolicy(bucket, policyAction, resource,
-			r.Referer(), handlers.GetSourceIP(r), r.URL.Query())
+	case authTypeSigned, authTypePresigned:
+		region := globalServerConfig.GetRegion()
+		switch action {
+		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
+			region = ""
+		}
+
+		if errorCode := isReqAuthenticated(r, region); errorCode != ErrNone {
+			return errorCode
+		}
+	default:
+		isOwner = false
+		accountName = ""
+	}
+
+	// LocationConstraint is valid only for CreateBucketAction.
+	var locationConstraint string
+	if action == policy.CreateBucketAction {
+		// To extract region from XML in request body, get copy of request body.
+		payload, err := ioutil.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return ErrMalformedXML
+		}
+
+		// Populate payload to extract location constraint.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+
+		var s3Error APIErrorCode
+		locationConstraint, s3Error = parseLocationConstraint(r)
+		if s3Error != ErrNone {
+			return s3Error
+		}
+
+		// Populate payload again to handle it in HTTP handler.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+	}
+
+	if globalPolicySys.IsAllowed(policy.Args{
+		AccountName:     accountName,
+		Action:          action,
+		BucketName:      bucketName,
+		ConditionValues: getConditionValues(r, locationConstraint),
+		IsOwner:         isOwner,
+		ObjectName:      objectName,
+	}) {
+		return ErrNone
 	}
 
-	// By default return ErrAccessDenied
 	return ErrAccessDenied
 }
 
@@ -166,59 +210,45 @@ func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode)
 
 // Verify if request has valid AWS Signature Version '4'.
 func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
-	if r == nil {
-		return ErrInternalError
-	}
-
 	if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
 		return errCode
 	}
 
-	payload, err := ioutil.ReadAll(r.Body)
+	var (
+		err                       error
+		contentMD5, contentSHA256 []byte
+	)
+	// Extract 'Content-Md5' if present.
+	if _, ok := r.Header["Content-Md5"]; ok {
+		contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get("Content-Md5"))
+		if err != nil || len(contentMD5) == 0 {
+			return ErrInvalidDigest
+		}
+	}
+
+	// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
+	// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
+	if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
+		if sha256Sum, ok := r.URL.Query()["X-Amz-Content-Sha256"]; ok && len(sha256Sum) > 0 {
+			contentSHA256, err = hex.DecodeString(sha256Sum[0])
+			if err != nil {
+				return ErrContentSHA256Mismatch
+			}
+		}
+	} else if _, ok := r.Header["X-Amz-Content-Sha256"]; !skipSHA256 && ok {
+		contentSHA256, err = hex.DecodeString(r.Header.Get("X-Amz-Content-Sha256"))
+		if err != nil || len(contentSHA256) == 0 {
+			return ErrContentSHA256Mismatch
+		}
+	}
+
+	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
+	// The verification happens implicit during reading.
+	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256))
 	if err != nil {
-		errorIf(err, "Unable to read request body for signature verification")
-		return ErrInternalError
-	}
-
-	// Populate back the payload.
-	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
-
-	// Verify Content-Md5, if payload is set.
-	if clntMD5B64, ok := r.Header["Content-Md5"]; ok {
-		if clntMD5B64[0] == "" {
-			return ErrInvalidDigest
-		}
-		md5Sum, err := base64.StdEncoding.Strict().DecodeString(clntMD5B64[0])
-		if err != nil {
-			return ErrInvalidDigest
-		}
-		if !bytes.Equal(md5Sum, getMD5Sum(payload)) {
-			return ErrBadDigest
-		}
-	}
-
-	if skipContentSha256Cksum(r) {
-		return ErrNone
-	}
-
-	// Verify that X-Amz-Content-Sha256 Header == sha256(payload)
-	// If X-Amz-Content-Sha256 header is not sent then we don't calculate/verify sha256(payload)
-	sumHex, ok := r.Header["X-Amz-Content-Sha256"]
-	if isRequestPresignedSignatureV4(r) {
-		sumHex, ok = r.URL.Query()["X-Amz-Content-Sha256"]
-	}
-	if ok {
-		if sumHex[0] == "" {
-			return ErrContentSHA256Mismatch
-		}
-		sum, err := hex.DecodeString(sumHex[0])
-		if err != nil {
-			return ErrContentSHA256Mismatch
-		}
-		if !bytes.Equal(sum, getSHA256Sum(payload)) {
-			return ErrContentSHA256Mismatch
-		}
+		return toAPIErrorCode(err)
 	}
+	r.Body = ioutil.NopCloser(reader)
 	return ErrNone
 }
 

cmd/auth-handler_test.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -361,8 +362,6 @@ func TestIsReqAuthenticated(t *testing.T) {
 		req     *http.Request
 		s3Error APIErrorCode
 	}{
-		// When request is nil, internal error is returned.
-		{nil, ErrInternalError},
 		// When request is unsigned, access denied is returned.
 		{mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrAccessDenied},
 		// Empty Content-Md5 header.
@@ -376,9 +375,11 @@ func TestIsReqAuthenticated(t *testing.T) {
 	}
 
 	// Validates all testcases.
-	for _, testCase := range testCases {
+	for i, testCase := range testCases {
 		if s3Error := isReqAuthenticated(testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
-			t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
+			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(err) != testCase.s3Error {
+				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %d)", i, testCase.s3Error, s3Error, toAPIErrorCode(err))
+			}
 		}
 	}
 }

cmd/auth-rpc-client.go

@@ -1,310 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bufio"
-	"crypto/tls"
-	"crypto/x509"
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"net/rpc"
-	"strings"
-	"sync"
-	"time"
-)
-
-// Attempt to retry only this many number of times before
-// giving up on the remote RPC entirely.
-const globalAuthRPCRetryThreshold = 1
-
-// authConfig requires to make new AuthRPCClient.
-type authConfig struct {
-	accessKey        string // Access key (like username) for authentication.
-	secretKey        string // Secret key (like Password) for authentication.
-	serverAddr       string // RPC server address.
-	serviceEndpoint  string // Endpoint on the server to make any RPC call.
-	secureConn       bool   // Make TLS connection to RPC server or not.
-	serviceName      string // Service name of auth server.
-	disableReconnect bool   // Disable reconnect on failure or not.
-
-	/// Retry configurable values.
-
-	// Each retry unit multiplicative, measured in time.Duration.
-	// This is the basic unit used for calculating backoffs.
-	retryUnit time.Duration
-	// Maximum retry duration i.e A caller would wait no more than this
-	// duration to continue their loop.
-	retryCap time.Duration
-
-	// Maximum retries an call authRPC client would do for a failed
-	// RPC call.
-	retryAttemptThreshold int
-}
-
-// AuthRPCClient is a authenticated RPC client which does authentication before doing Call().
-type AuthRPCClient struct {
-	sync.RWMutex             // Mutex to lock this object.
-	rpcClient    *rpc.Client // RPC client to make any RPC call.
-	config       authConfig  // Authentication configuration information.
-	authToken    string      // Authentication token.
-	version      semVersion  // RPC version.
-}
-
-// newAuthRPCClient - returns a JWT based authenticated (go) rpc client, which does automatic reconnect.
-func newAuthRPCClient(config authConfig) *AuthRPCClient {
-	// Check if retry params are set properly if not default them.
-	emptyDuration := time.Duration(int64(0))
-	if config.retryUnit == emptyDuration {
-		config.retryUnit = defaultRetryUnit
-	}
-	if config.retryCap == emptyDuration {
-		config.retryCap = defaultRetryCap
-	}
-	if config.retryAttemptThreshold == 0 {
-		config.retryAttemptThreshold = globalAuthRPCRetryThreshold
-	}
-
-	return &AuthRPCClient{
-		config:  config,
-		version: globalRPCAPIVersion,
-	}
-}
-
-// Login a JWT based authentication is performed with rpc server.
-func (authClient *AuthRPCClient) Login() (err error) {
-	// Login should be attempted one at a time.
-	//
-	// The reason for large region lock here is
-	// to avoid two simultaneous login attempts
-	// racing over each other.
-	//
-	// #1 Login() gets the lock proceeds to login.
-	// #2 Login() waits for the unlock to happen
-	//    after login in #1.
-	// #1 Successfully completes login saves the
-	//    newly acquired token.
-	// #2 Successfully gets the lock and proceeds,
-	//    but since we have acquired the token
-	//    already the call quickly returns.
-	authClient.Lock()
-	defer authClient.Unlock()
-
-	// Attempt to login if not logged in already.
-	if authClient.authToken == "" {
-		var authToken string
-		authToken, err = authenticateNode(authClient.config.accessKey, authClient.config.secretKey)
-		if err != nil {
-			return err
-		}
-
-		// Login to authenticate your token.
-		var (
-			loginMethod = authClient.config.serviceName + loginMethodName
-			loginArgs   = LoginRPCArgs{
-				AuthToken:   authToken,
-				Version:     globalRPCAPIVersion,
-				RequestTime: UTCNow(),
-			}
-		)
-
-		// Re-dial after we have disconnected or if its a fresh run.
-		var rpcClient *rpc.Client
-		rpcClient, err = rpcDial(authClient.config.serverAddr, authClient.config.serviceEndpoint, authClient.config.secureConn)
-		if err != nil {
-			return err
-		}
-
-		if err = rpcClient.Call(loginMethod, &loginArgs, &LoginRPCReply{}); err != nil {
-			// gob doesn't provide any typed errors for us to reflect
-			// upon, this is the only way to return proper error.
-			if strings.Contains(err.Error(), "gob: wrong type") {
-				return errRPCAPIVersionUnsupported
-			}
-			return err
-		}
-
-		// Initialize rpc client and auth token after a successful login.
-		authClient.authToken = authToken
-		authClient.rpcClient = rpcClient
-	}
-
-	return nil
-}
-
-// call makes a RPC call after logs into the server.
-func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
-	SetAuthToken(authToken string)
-	SetRPCAPIVersion(version semVersion)
-}, reply interface{}) (err error) {
-	if err = authClient.Login(); err != nil {
-		return err
-	} // On successful login, execute RPC call.
-
-	// Set token before the rpc call.
-	authClient.RLock()
-	defer authClient.RUnlock()
-	args.SetAuthToken(authClient.authToken)
-	args.SetRPCAPIVersion(authClient.version)
-
-	// Do an RPC call.
-	return authClient.rpcClient.Call(serviceMethod, args, reply)
-}
-
-// Call executes RPC call till success or globalAuthRPCRetryThreshold on ErrShutdown.
-func (authClient *AuthRPCClient) Call(serviceMethod string, args interface {
-	SetAuthToken(authToken string)
-	SetRPCAPIVersion(version semVersion)
-}, reply interface{}) (err error) {
-
-	// Done channel is used to close any lingering retry routine, as soon
-	// as this function returns.
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
-	for i := range newRetryTimer(authClient.config.retryUnit, authClient.config.retryCap, doneCh) {
-		if err = authClient.call(serviceMethod, args, reply); err == rpc.ErrShutdown {
-			// As connection at server side is closed, close the rpc client.
-			authClient.Close()
-
-			// Retry if reconnect is not disabled.
-			if !authClient.config.disableReconnect {
-				// Retry until threshold reaches.
-				if i < authClient.config.retryAttemptThreshold {
-					continue
-				}
-			}
-		}
-		// gob doesn't provide any typed errors for us to reflect
-		// upon, this is the only way to return proper error.
-		if err != nil && strings.Contains(err.Error(), "gob: wrong type") {
-			err = errRPCAPIVersionUnsupported
-		}
-		break
-	}
-	return err
-}
-
-// Close closes underlying RPC Client.
-func (authClient *AuthRPCClient) Close() error {
-	authClient.Lock()
-	defer authClient.Unlock()
-
-	if authClient.rpcClient == nil {
-		return nil
-	}
-
-	authClient.authToken = ""
-	return authClient.rpcClient.Close()
-}
-
-// ServerAddr returns the serverAddr (network address) of the connection.
-func (authClient *AuthRPCClient) ServerAddr() string {
-	return authClient.config.serverAddr
-}
-
-// ServiceEndpoint returns the RPC service endpoint of the connection.
-func (authClient *AuthRPCClient) ServiceEndpoint() string {
-	return authClient.config.serviceEndpoint
-}
-
-// default Dial timeout for RPC connections.
-const defaultDialTimeout = 3 * time.Second
-
-// Connect success message required from rpc server.
-const connectSuccessMessage = "200 Connected to Go RPC"
-
-// dial tries to establish a connection to serverAddr in a safe manner.
-// If there is a valid rpc.Cliemt, it returns that else creates a new one.
-func rpcDial(serverAddr, serviceEndpoint string, secureConn bool) (netRPCClient *rpc.Client, err error) {
-	if serverAddr == "" || serviceEndpoint == "" {
-		return nil, errInvalidArgument
-	}
-	d := &net.Dialer{
-		Timeout: defaultDialTimeout,
-	}
-	var conn net.Conn
-	if secureConn {
-		var hostname string
-		if hostname, _, err = net.SplitHostPort(serverAddr); err != nil {
-			return nil, &net.OpError{
-				Op:   "dial-http",
-				Net:  serverAddr + serviceEndpoint,
-				Addr: nil,
-				Err:  fmt.Errorf("Unable to parse server address <%s>: %s", serverAddr, err),
-			}
-		}
-		// ServerName in tls.Config needs to be specified to support SNI certificates.
-		conn, err = tls.DialWithDialer(d, "tcp", serverAddr, &tls.Config{
-			ServerName: hostname,
-			RootCAs:    globalRootCAs,
-		})
-	} else {
-		conn, err = d.Dial("tcp", serverAddr)
-	}
-
-	if err != nil {
-		// Print RPC connection errors that are worthy to display in log.
-		switch err.(type) {
-		case x509.HostnameError:
-			errorIf(err, "Unable to establish secure connection to %s", serverAddr)
-		}
-
-		return nil, &net.OpError{
-			Op:   "dial-http",
-			Net:  serverAddr + serviceEndpoint,
-			Addr: nil,
-			Err:  err,
-		}
-	}
-
-	// Check for network errors writing over the dialed conn.
-	if _, err = io.WriteString(conn, "CONNECT "+serviceEndpoint+" HTTP/1.0\n\n"); err != nil {
-		conn.Close()
-		return nil, &net.OpError{
-			Op:   "dial-http",
-			Net:  serverAddr + serviceEndpoint,
-			Addr: nil,
-			Err:  err,
-		}
-	}
-
-	// Attempt to read the HTTP response for the HTTP method CONNECT, upon
-	// success return the RPC connection instance.
-	resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{
-		Method: http.MethodConnect,
-	})
-	if err != nil {
-		conn.Close()
-		return nil, &net.OpError{
-			Op:   "dial-http",
-			Net:  serverAddr + serviceEndpoint,
-			Addr: nil,
-			Err:  err,
-		}
-	}
-	if resp.Status != connectSuccessMessage {
-		conn.Close()
-		return nil, errors.New("unexpected HTTP response: " + resp.Status)
-	}
-
-	// Initialize rpc client.
-	return rpc.NewClient(conn), nil
-}

cmd/auth-rpc-client_test.go

@@ -1,138 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/x509"
-	"os"
-	"path"
-	"testing"
-)
-
-// Tests authorized RPC client.
-func TestAuthRPCClient(t *testing.T) {
-	// reset globals.
-	// this is to make sure that the tests are not affected by modified globals.
-	resetTestGlobals()
-
-	authCfg := authConfig{
-		accessKey:       "123",
-		secretKey:       "123",
-		serverAddr:      "localhost:9000",
-		serviceEndpoint: "/rpc/disk",
-		secureConn:      false,
-		serviceName:     "MyPackage",
-	}
-	authRPC := newAuthRPCClient(authCfg)
-	if authRPC.ServerAddr() != authCfg.serverAddr {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServerAddr(), authCfg.serverAddr)
-	}
-	if authRPC.ServiceEndpoint() != authCfg.serviceEndpoint {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServiceEndpoint(), authCfg.serviceEndpoint)
-	}
-	authCfg = authConfig{
-		accessKey:   "123",
-		secretKey:   "123",
-		secureConn:  false,
-		serviceName: "MyPackage",
-	}
-	authRPC = newAuthRPCClient(authCfg)
-	if authRPC.ServerAddr() != authCfg.serverAddr {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServerAddr(), authCfg.serverAddr)
-	}
-	if authRPC.ServiceEndpoint() != authCfg.serviceEndpoint {
-		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServiceEndpoint(), authCfg.serviceEndpoint)
-	}
-}
-
-// Test rpc dial test.
-func TestRPCDial(t *testing.T) {
-	prevRootCAs := globalRootCAs
-	defer func() {
-		globalRootCAs = prevRootCAs
-	}()
-
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	testServer := StartTestServer(t, "")
-	defer testServer.Stop()
-
-	cert, key, err := generateTLSCertKey("127.0.0.1")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Set global root CAs.
-	globalRootCAs = x509.NewCertPool()
-	globalRootCAs.AppendCertsFromPEM(cert)
-
-	testServerTLS := StartTestTLSServer(t, "", cert, key)
-	defer testServerTLS.Stop()
-
-	adminEndpoint := path.Join(minioReservedBucketPath, adminPath)
-	testCases := []struct {
-		serverAddr     string
-		serverEndpoint string
-		success        bool
-		secure         bool
-	}{
-		// Empty server addr should fail.
-		{
-			serverAddr:     "",
-			serverEndpoint: adminEndpoint,
-			success:        false,
-		},
-		// Unexpected server addr should fail.
-		{
-			serverAddr:     "example.com",
-			serverEndpoint: adminEndpoint,
-			success:        false,
-		},
-		// Server addr connects but fails for CONNECT call.
-		{
-			serverAddr:     "example.com:80",
-			serverEndpoint: "/",
-			success:        false,
-		},
-		// Successful connecting to insecure RPC server.
-		{
-			serverAddr:     testServer.Server.Listener.Addr().String(),
-			serverEndpoint: adminEndpoint,
-			success:        true,
-		},
-		// Successful connecting to secure RPC server.
-		{
-			serverAddr:     testServerTLS.Server.Listener.Addr().String(),
-			serverEndpoint: adminEndpoint,
-			success:        true,
-			secure:         true,
-		},
-	}
-	for i, testCase := range testCases {
-		_, err = rpcDial(testCase.serverAddr, testCase.serverEndpoint, testCase.secure)
-		if err != nil && testCase.success {
-			t.Errorf("Test %d: Expected success but found failure instead %s", i+1, err)
-		}
-		if err == nil && !testCase.success {
-			t.Errorf("Test %d: Expected failure but found success instead", i+1)
-		}
-	}
-}

cmd/auth-rpc-server_test.go

@@ -1,88 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"os"
-	"testing"
-	"time"
-)
-
-func TestLogin(t *testing.T) {
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatalf("Failed to create test config - %v", err)
-	}
-	defer os.RemoveAll(rootPath)
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ls := AuthRPCServer{}
-	testCases := []struct {
-		args        LoginRPCArgs
-		skewTime    time.Duration
-		expectedErr error
-	}{
-		// Valid case.
-		{
-			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   globalRPCAPIVersion,
-			},
-			skewTime:    0,
-			expectedErr: nil,
-		},
-		// Valid username, password and request time, not version.
-		{
-			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   semVersion{1, 0, 0},
-			},
-			skewTime:    0,
-			expectedErr: errRPCAPIVersionUnsupported,
-		},
-		// Valid username, password and version, not request time
-		{
-			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   globalRPCAPIVersion,
-			},
-			skewTime:    20 * time.Minute,
-			expectedErr: errServerTimeMismatch,
-		},
-		// Invalid token, fails with authentication error
-		{
-			args: LoginRPCArgs{
-				AuthToken: "",
-				Version:   globalRPCAPIVersion,
-			},
-			skewTime:    0,
-			expectedErr: errAuthentication,
-		},
-	}
-	for i, test := range testCases {
-		reply := LoginRPCReply{}
-		test.args.RequestTime = UTCNow().Add(test.skewTime)
-		err := ls.Login(&test.args, &reply)
-		if err != test.expectedErr {
-			t.Errorf("Test %d: Expected error %v but received %v",
-				i+1, test.expectedErr, err)
-		}
-	}
-}

cmd/bool-flag.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,11 +21,11 @@ import (
 	"fmt"
 )
 
-// BrowserFlag - wrapper bool type.
-type BrowserFlag bool
+// BoolFlag - wrapper bool type.
+type BoolFlag bool
 
-// String - returns string of BrowserFlag.
-func (bf BrowserFlag) String() string {
+// String - returns string of BoolFlag.
+func (bf BoolFlag) String() string {
 	if bf {
 		return "on"
 	}
@@ -33,20 +33,20 @@ func (bf BrowserFlag) String() string {
 	return "off"
 }
 
-// MarshalJSON - converts BrowserFlag into JSON data.
-func (bf BrowserFlag) MarshalJSON() ([]byte, error) {
+// MarshalJSON - converts BoolFlag into JSON data.
+func (bf BoolFlag) MarshalJSON() ([]byte, error) {
 	return json.Marshal(bf.String())
 }
 
-// UnmarshalJSON - parses given data into BrowserFlag.
-func (bf *BrowserFlag) UnmarshalJSON(data []byte) (err error) {
+// UnmarshalJSON - parses given data into BoolFlag.
+func (bf *BoolFlag) UnmarshalJSON(data []byte) (err error) {
 	var s string
 	if err = json.Unmarshal(data, &s); err == nil {
-		b := BrowserFlag(true)
+		b := BoolFlag(true)
 		if s == "" {
 			// Empty string is treated as valid.
 			*bf = b
-		} else if b, err = ParseBrowserFlag(s); err == nil {
+		} else if b, err = ParseBoolFlag(s); err == nil {
 			*bf = b
 		}
 	}
@@ -54,14 +54,14 @@ func (bf *BrowserFlag) UnmarshalJSON(data []byte) (err error) {
 	return err
 }
 
-// ParseBrowserFlag - parses string into BrowserFlag.
-func ParseBrowserFlag(s string) (bf BrowserFlag, err error) {
+// ParseBoolFlag - parses string into BoolFlag.
+func ParseBoolFlag(s string) (bf BoolFlag, err error) {
 	if s == "on" {
 		bf = true
 	} else if s == "off" {
 		bf = false
 	} else {
-		err = fmt.Errorf("invalid value ‘%s’ for BrowserFlag", s)
+		err = fmt.Errorf("invalid value ‘%s’ for BoolFlag", s)
 	}
 
 	return bf, err

cmd/bool-flag_test.go

@@ -21,17 +21,17 @@ import (
 	"testing"
 )
 
-// Test BrowserFlag.String()
-func TestBrowserFlagString(t *testing.T) {
-	var bf BrowserFlag
+// Test BoolFlag.String()
+func TestBoolFlagString(t *testing.T) {
+	var bf BoolFlag
 
 	testCases := []struct {
-		flag           BrowserFlag
+		flag           BoolFlag
 		expectedResult string
 	}{
 		{bf, "off"},
-		{BrowserFlag(true), "on"},
-		{BrowserFlag(false), "off"},
+		{BoolFlag(true), "on"},
+		{BoolFlag(false), "off"},
 	}
 
 	for _, testCase := range testCases {
@@ -42,17 +42,17 @@ func TestBrowserFlagString(t *testing.T) {
 	}
 }
 
-// Test BrowserFlag.MarshalJSON()
-func TestBrowserFlagMarshalJSON(t *testing.T) {
-	var bf BrowserFlag
+// Test BoolFlag.MarshalJSON()
+func TestBoolFlagMarshalJSON(t *testing.T) {
+	var bf BoolFlag
 
 	testCases := []struct {
-		flag           BrowserFlag
+		flag           BoolFlag
 		expectedResult string
 	}{
 		{bf, `"off"`},
-		{BrowserFlag(true), `"on"`},
-		{BrowserFlag(false), `"off"`},
+		{BoolFlag(true), `"on"`},
+		{BoolFlag(false), `"off"`},
 	}
 
 	for _, testCase := range testCases {
@@ -63,27 +63,27 @@ func TestBrowserFlagMarshalJSON(t *testing.T) {
 	}
 }
 
-// Test BrowserFlag.UnmarshalJSON()
-func TestBrowserFlagUnmarshalJSON(t *testing.T) {
+// Test BoolFlag.UnmarshalJSON()
+func TestBoolFlagUnmarshalJSON(t *testing.T) {
 	testCases := []struct {
 		data           []byte
-		expectedResult BrowserFlag
+		expectedResult BoolFlag
 		expectedErr    error
 	}{
-		{[]byte(`{}`), BrowserFlag(false), errors.New("json: cannot unmarshal object into Go value of type string")},
-		{[]byte(`["on"]`), BrowserFlag(false), errors.New("json: cannot unmarshal array into Go value of type string")},
-		{[]byte(`"junk"`), BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
-		{[]byte(`"true"`), BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
-		{[]byte(`"false"`), BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
-		{[]byte(`"ON"`), BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
-		{[]byte(`"OFF"`), BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
-		{[]byte(`""`), BrowserFlag(true), nil},
-		{[]byte(`"on"`), BrowserFlag(true), nil},
-		{[]byte(`"off"`), BrowserFlag(false), nil},
+		{[]byte(`{}`), BoolFlag(false), errors.New("json: cannot unmarshal object into Go value of type string")},
+		{[]byte(`["on"]`), BoolFlag(false), errors.New("json: cannot unmarshal array into Go value of type string")},
+		{[]byte(`"junk"`), BoolFlag(false), errors.New("invalid value ‘junk’ for BoolFlag")},
+		{[]byte(`"true"`), BoolFlag(false), errors.New("invalid value ‘true’ for BoolFlag")},
+		{[]byte(`"false"`), BoolFlag(false), errors.New("invalid value ‘false’ for BoolFlag")},
+		{[]byte(`"ON"`), BoolFlag(false), errors.New("invalid value ‘ON’ for BoolFlag")},
+		{[]byte(`"OFF"`), BoolFlag(false), errors.New("invalid value ‘OFF’ for BoolFlag")},
+		{[]byte(`""`), BoolFlag(true), nil},
+		{[]byte(`"on"`), BoolFlag(true), nil},
+		{[]byte(`"off"`), BoolFlag(false), nil},
 	}
 
 	for _, testCase := range testCases {
-		var flag BrowserFlag
+		var flag BoolFlag
 		err := (&flag).UnmarshalJSON(testCase.data)
 		if testCase.expectedErr == nil {
 			if err != nil {
@@ -101,25 +101,25 @@ func TestBrowserFlagUnmarshalJSON(t *testing.T) {
 	}
 }
 
-// Test ParseBrowserFlag()
-func TestParseBrowserFlag(t *testing.T) {
+// Test ParseBoolFlag()
+func TestParseBoolFlag(t *testing.T) {
 	testCases := []struct {
 		flagStr        string
-		expectedResult BrowserFlag
+		expectedResult BoolFlag
 		expectedErr    error
 	}{
-		{"", BrowserFlag(false), errors.New("invalid value ‘’ for BrowserFlag")},
-		{"junk", BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
-		{"true", BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
-		{"false", BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
-		{"ON", BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
-		{"OFF", BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
-		{"on", BrowserFlag(true), nil},
-		{"off", BrowserFlag(false), nil},
+		{"", BoolFlag(false), errors.New("invalid value ‘’ for BoolFlag")},
+		{"junk", BoolFlag(false), errors.New("invalid value ‘junk’ for BoolFlag")},
+		{"true", BoolFlag(false), errors.New("invalid value ‘true’ for BoolFlag")},
+		{"false", BoolFlag(false), errors.New("invalid value ‘false’ for BoolFlag")},
+		{"ON", BoolFlag(false), errors.New("invalid value ‘ON’ for BoolFlag")},
+		{"OFF", BoolFlag(false), errors.New("invalid value ‘OFF’ for BoolFlag")},
+		{"on", BoolFlag(true), nil},
+		{"off", BoolFlag(false), nil},
 	}
 
 	for _, testCase := range testCases {
-		bf, err := ParseBrowserFlag(testCase.flagStr)
+		bf, err := ParseBoolFlag(testCase.flagStr)
 		if testCase.expectedErr == nil {
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)

cmd/browser-peer-rpc.go

@@ -1,140 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"fmt"
-	"path"
-	"sync"
-	"time"
-
-	"github.com/minio/minio/pkg/auth"
-)
-
-// SetAuthPeerArgs - Arguments collection for SetAuth RPC call
-type SetAuthPeerArgs struct {
-	// For Auth
-	AuthRPCArgs
-
-	// New credentials that receiving peer should update to.
-	Creds auth.Credentials
-}
-
-// SetAuthPeer - Update to new credentials sent from a peer Minio
-// server. Since credentials are already validated on the sending
-// peer, here we just persist to file and update in-memory config. All
-// subsequently running isAuthTokenValid() calls will fail, and clients
-// will be forced to re-establish connections. Connections will be
-// re-established only when the sending client has also updated its
-// credentials.
-func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthRPCReply) error {
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-
-	if !args.Creds.IsValid() {
-		return fmt.Errorf("Invalid credential passed")
-	}
-
-	// Acquire lock before updating global configuration.
-	globalServerConfigMu.Lock()
-	defer globalServerConfigMu.Unlock()
-
-	// Update credentials in memory
-	prevCred := globalServerConfig.SetCredential(args.Creds)
-
-	// Save credentials to config file
-	if err := globalServerConfig.Save(); err != nil {
-		// Save the current creds when failed to update.
-		globalServerConfig.SetCredential(prevCred)
-
-		errorIf(err, "Unable to update the config with new credentials sent from browser RPC.")
-		return err
-	}
-
-	return nil
-}
-
-// Sends SetAuthPeer RPCs to all peers in the Minio cluster
-func updateCredsOnPeers(creds auth.Credentials) map[string]error {
-	peers := GetRemotePeers(globalEndpoints)
-
-	// Array of errors for each peer
-	errs := make([]error, len(peers))
-	var wg sync.WaitGroup
-
-	serverCred := globalServerConfig.GetCredential()
-	// Launch go routines to send request to each peer in parallel.
-	for ix := range peers {
-		wg.Add(1)
-		go func(ix int) {
-			defer wg.Done()
-
-			// Exclude self to avoid race with
-			// invalidating the RPC token.
-			if peers[ix] == globalMinioAddr {
-				errs[ix] = nil
-				return
-			}
-
-			// Initialize client
-			client := newAuthRPCClient(authConfig{
-				accessKey:       serverCred.AccessKey,
-				secretKey:       serverCred.SecretKey,
-				serverAddr:      peers[ix],
-				secureConn:      globalIsSSL,
-				serviceEndpoint: path.Join(minioReservedBucketPath, browserPeerPath),
-				serviceName:     "BrowserPeer",
-			})
-
-			// Construct RPC call arguments.
-			args := SetAuthPeerArgs{Creds: creds}
-
-			// Make RPC call - we only care about error
-			// response and not the reply.
-			err := client.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{})
-
-			// We try a bit hard (3 attempts with 1 second delay)
-			// to set creds on peers in case of failure.
-			if err != nil {
-				for i := 0; i < 2; i++ {
-					time.Sleep(1 * time.Second) // 1 second delay.
-					err = client.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{})
-					if err == nil {
-						break
-					}
-				}
-			}
-
-			// Send result down the channel
-			errs[ix] = err
-		}(ix)
-	}
-
-	// Wait for requests to complete.
-	wg.Wait()
-
-	// Put errors into map.
-	errsMap := make(map[string]error)
-	for i, err := range errs {
-		if err != nil {
-			errsMap[peers[i]] = err
-		}
-	}
-
-	return errsMap
-}

cmd/browser-peer-rpc_test.go

@@ -1,116 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"path"
-	"testing"
-
-	"github.com/minio/minio/pkg/auth"
-)
-
-// API suite container common to both FS and XL.
-type TestRPCBrowserPeerSuite struct {
-	serverType   string
-	testServer   TestServer
-	testAuthConf authConfig
-}
-
-// Setting up the test suite and starting the Test server.
-func (s *TestRPCBrowserPeerSuite) SetUpSuite(t *testing.T) {
-	s.testServer = StartTestBrowserPeerRPCServer(t, s.serverType)
-	s.testAuthConf = authConfig{
-		serverAddr:      s.testServer.Server.Listener.Addr().String(),
-		accessKey:       s.testServer.AccessKey,
-		secretKey:       s.testServer.SecretKey,
-		serviceEndpoint: path.Join(minioReservedBucketPath, browserPeerPath),
-		serviceName:     "BrowserPeer",
-	}
-}
-
-// TeatDownSuite - called implicitly by after all tests are run in
-// browser peer rpc suite.
-func (s *TestRPCBrowserPeerSuite) TearDownSuite(t *testing.T) {
-	s.testServer.Stop()
-}
-
-func TestBrowserPeerRPC(t *testing.T) {
-	// setup code
-	s := &TestRPCBrowserPeerSuite{serverType: "XL"}
-	s.SetUpSuite(t)
-
-	// run test
-	s.testBrowserPeerRPC(t)
-
-	// teardown code
-	s.TearDownSuite(t)
-}
-
-// Tests for browser peer rpc.
-func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
-	// Construct RPC call arguments.
-	creds, err := auth.CreateCredentials("abcd1", "abcd1234")
-	if err != nil {
-		t.Fatalf("unable to create credential. %v", err)
-	}
-
-	// Validate for invalid token.
-	args := SetAuthPeerArgs{Creds: creds}
-	rclient := newAuthRPCClient(s.testAuthConf)
-	defer rclient.Close()
-	if err = rclient.Login(); err != nil {
-		t.Fatal(err)
-	}
-	rclient.authToken = "garbage"
-	if err = rclient.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{}); err != nil {
-		if err.Error() != errInvalidToken.Error() {
-			t.Fatal(err)
-		}
-	}
-
-	// Validate for successful Peer update.
-	args = SetAuthPeerArgs{Creds: creds}
-	client := newAuthRPCClient(s.testAuthConf)
-	defer client.Close()
-	err = client.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Validate for failure in login handler with previous credentials.
-	rclient = newAuthRPCClient(s.testAuthConf)
-	defer rclient.Close()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	rclient.authToken = token
-	if err = rclient.Login(); err != nil {
-		if err.Error() != errInvalidAccessKeyID.Error() {
-			t.Fatal(err)
-		}
-	}
-
-	token, err = authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	rclient.authToken = token
-	if err = rclient.Login(); err != nil {
-		t.Fatal(err)
-	}
-}

cmd/browser-rpc-router.go

@@ -1,51 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	router "github.com/gorilla/mux"
-
-	"github.com/minio/minio/pkg/errors"
-)
-
-// Set up an RPC endpoint that receives browser related calls. The
-// original motivation is for propagating credentials change
-// throughout Minio cluster, initiated from a Minio browser session.
-
-const (
-	browserPeerPath = "/browser/setauth"
-)
-
-// The Type exporting methods exposed for RPC calls.
-type browserPeerAPIHandlers struct {
-	AuthRPCServer
-}
-
-// Register RPC router
-func registerBrowserPeerRPCRouter(mux *router.Router) error {
-	bpHandlers := &browserPeerAPIHandlers{AuthRPCServer{}}
-
-	bpRPCServer := newRPCServer()
-	err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)
-	if err != nil {
-		return errors.Trace(err)
-	}
-
-	bpRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
-	bpRouter.Path(browserPeerPath).Handler(bpRPCServer)
-	return nil
-}

cmd/bucket-handlers-listobjects.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // Validate all the ListObjects query arguments, returns an APIErrorCode
@@ -64,25 +65,24 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+	urlValues := r.URL.Query()
 
-	// In ListObjectsV2 'continuation-token' is the marker.
-	marker := token
-	// Check if 'continuation-token' is empty.
-	if token == "" {
-		// Then we need to use 'start-after' as marker instead.
-		marker = startAfter
+	// Extract all the listObjectsV2 query params to their native values.
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
+
+	if errCode != ErrNone {
+		writeErrorResponse(w, errCode, r.URL)
+		return
 	}
 
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -92,8 +92,8 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
-	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, fetchOwner, startAfter)
+	// marshaled into S3 compatible XML header.
+	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -134,7 +134,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -158,7 +158,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
+	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/bucket-handlers.go

@@ -20,89 +20,67 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/xml"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/url"
 	"path"
 	"path/filepath"
-	"reflect"
 	"strings"
 	"sync"
 
-	mux "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/policy"
+	"github.com/gorilla/mux"
+
 	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
-// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-// Enforces bucket policies for a bucket for a given tatusaction.
-func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, queryParams url.Values) (s3Error APIErrorCode) {
-	// Verify if bucket actually exists
-	objAPI := newObjectLayerFn()
-	if err := checkBucketExist(bucket, objAPI); err != nil {
-		err = errors.Cause(err)
-		switch err.(type) {
-		case BucketNameInvalid:
-			// Return error for invalid bucket name.
-			return ErrInvalidBucketName
-		case BucketNotFound:
-			// For no bucket found we return NoSuchBucket instead.
-			return ErrNoSuchBucket
+// Check if there are buckets on server without corresponding entry in etcd backend and
+// make entries. Here is the general flow
+// - Range over all the available buckets
+// - Check if a bucket has an entry in etcd backend
+// -- If no, make an entry
+// -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
+// -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
+func initFederatorBackend(objLayer ObjectLayer) {
+	b, err := objLayer.ListBuckets(context.Background())
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return
+	}
+
+	g := errgroup.WithNErrs(len(b))
+	for index := range b {
+		index := index
+		g.Go(func() error {
+			r, gerr := globalDNSConfig.Get(b[index].Name)
+			if gerr != nil {
+				if gerr == dns.ErrNoEntriesFound {
+					return globalDNSConfig.Put(b[index].Name)
+				}
+				return gerr
+			}
+			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				// There is already an entry for this bucket, with all IP addresses different. This indicates a bucket name collision. Log an error and continue.
+				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", b[index].Name, globalDomainIPs.ToSlice())
+			}
+			return nil
+		}, index)
+	}
+
+	for _, err := range g.Wait() {
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			return
 		}
-		errorIf(err, "Unable to read bucket policy.")
-		// Return internal error for any other errors so that we can investigate.
-		return ErrInternalError
 	}
-
-	// Fetch bucket policy, if policy is not set return access denied.
-	p, err := objAPI.GetBucketPolicy(context.Background(), bucket)
-	if err != nil {
-		return ErrAccessDenied
-	}
-	if reflect.DeepEqual(p, emptyBucketPolicy) {
-		return ErrAccessDenied
-	}
-
-	// Construct resource in 'arn:aws:s3:::examplebucket/object' format.
-	arn := bucketARNPrefix + strings.TrimSuffix(strings.TrimPrefix(resource, "/"), "/")
-
-	// Get conditions for policy verification.
-	conditionKeyMap := make(policy.ConditionKeyMap)
-	for queryParam := range queryParams {
-		conditionKeyMap[queryParam] = set.CreateStringSet(queryParams.Get(queryParam))
-	}
-
-	// Add request referer to conditionKeyMap if present.
-	if referer != "" {
-		conditionKeyMap["referer"] = set.CreateStringSet(referer)
-	}
-	// Add request source Ip to conditionKeyMap.
-	conditionKeyMap["ip"] = set.CreateStringSet(sourceIP)
-
-	// Validate action, resource and conditions with current policy statements.
-	if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, p.Statements) {
-		return ErrAccessDenied
-	}
-	return ErrNone
-}
-
-// Check if the action is allowed on the bucket/prefix.
-func isBucketActionAllowed(action, bucket, prefix string, objectAPI ObjectLayer) bool {
-	bp, err := objectAPI.GetBucketPolicy(context.Background(), bucket)
-	if err != nil {
-		return false
-	}
-	if reflect.DeepEqual(bp, emptyBucketPolicy) {
-		return false
-	}
-	resource := bucketARNPrefix + path.Join(bucket, prefix)
-	var conditionKeyMap map[string]set.StringSet
-	// Validate action, resource and conditions with current policy statements.
-	return bucketPolicyEvalStatements(action, resource, conditionKeyMap, bp.Statements)
 }
 
 // GetBucketLocationHandler - GET Bucket location.
@@ -120,12 +98,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	s3Error := checkRequestAuthType(r, bucket, "s3:GetBucketLocation", globalMinioDefaultRegion)
-	if s3Error == ErrInvalidRegion {
-		// Clients like boto3 send getBucketLocation() call signed with region that is configured.
-		s3Error = checkRequestAuthType(r, "", "s3:GetBucketLocation", globalServerConfig.GetRegion())
-	}
-	if s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -179,7 +152,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucketMultipartUploads", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -227,21 +200,38 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	if api.CacheAPI() != nil {
 		listBuckets = api.CacheAPI().ListBuckets
 	}
-	// ListBuckets does not have any bucket action.
-	s3Error := checkRequestAuthType(r, "", "", globalMinioDefaultRegion)
-	if s3Error == ErrInvalidRegion {
-		// Clients like boto3 send listBuckets() call signed with region that is configured.
-		s3Error = checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
-	}
-	if s3Error != ErrNone {
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
-	// Invoke the list buckets.
-	bucketsInfo, err := listBuckets(ctx)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
+	// If etcd, dns federation configured list buckets from etcd.
+	var bucketsInfo []BucketInfo
+	if globalDNSConfig != nil {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil && err != dns.ErrNoEntriesFound {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+		bucketSet := set.NewStringSet()
+		for _, dnsRecord := range dnsBuckets {
+			if bucketSet.Contains(dnsRecord.Key) {
+				continue
+			}
+			bucketsInfo = append(bucketsInfo, BucketInfo{
+				Name:    strings.Trim(dnsRecord.Key, slashSeparator),
+				Created: dnsRecord.CreationDate,
+			})
+			bucketSet.Add(dnsRecord.Key)
+		}
+	} else {
+		// Invoke the list buckets.
+		var err error
+		bucketsInfo, err = listBuckets(ctx)
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	// Generate response.
@@ -265,12 +255,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var authError APIErrorCode
-	if authError = checkRequestAuthType(r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); authError != ErrNone {
+	var s3Error APIErrorCode
+	if s3Error = checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, ""); s3Error != ErrNone {
 		// In the event access is denied, a 200 response should still be returned
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-		if authError != ErrAccessDenied {
-			writeErrorResponse(w, authError, r.URL)
+		if s3Error != ErrAccessDenied {
+			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	}
@@ -290,11 +280,17 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	// Allocate incoming content length bytes.
-	deleteXMLBytes := make([]byte, r.ContentLength)
+	var deleteXMLBytes []byte
+	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
+	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
+		deleteXMLBytes = make([]byte, maxBodySize)
+	} else {
+		deleteXMLBytes = make([]byte, r.ContentLength)
+	}
 
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
-		errorIf(err, "Unable to read HTTP body.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
@@ -302,7 +298,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Unmarshal list of keys to be deleted.
 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
-		errorIf(err, "Unable to unmarshal delete objects request XML.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, ErrMalformedXML, r.URL)
 		return
 	}
@@ -325,7 +321,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			defer wg.Done()
 			// If the request is denied access, each item
 			// should be marked as 'AccessDenied'
-			if authError == ErrAccessDenied {
+			if s3Error == ErrAccessDenied {
 				dErrs[i] = PrefixAccessDenied{
 					Bucket: bucket,
 					Object: obj.ObjectName,
@@ -354,7 +350,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			deletedObjects = append(deletedObjects, object)
 			continue
 		}
-		if _, ok := errors.Cause(err).(ObjectNotFound); ok {
+		if _, ok := err.(ObjectNotFound); ok {
 			// If the object is not found it should be
 			// accounted as deleted as per S3 spec.
 			deletedObjects = append(deletedObjects, object)
@@ -377,7 +373,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Get host and port from Request.RemoteAddr failing which
 	// fill them with empty strings.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -410,16 +406,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// PutBucket does not have any bucket action.
-	s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
-	if s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
@@ -434,12 +428,33 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	if globalDNSConfig != nil {
+		if _, err := globalDNSConfig.Get(bucket); err != nil {
+			if err == dns.ErrNoEntriesFound {
+				// Proceed to creating a bucket.
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
+				if err = globalDNSConfig.Put(bucket); err != nil {
+					objectAPI.DeleteBucket(ctx, bucket)
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
+
+				// Make sure to add Location information here only for bucket
+				w.Header().Set("Location", getObjectLocation(r, globalDomainName, bucket, ""))
+
+				writeSuccessResponseHeadersOnly(w)
+				return
+			}
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+
+		}
+		writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL)
 		return
 	}
-	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
@@ -490,7 +505,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// be loaded in memory, the remaining being put in temporary files.
 	reader, err := r.MultipartReader()
 	if err != nil {
-		errorIf(err, "Unable to initialize multipart reader.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
 		return
 	}
@@ -498,7 +513,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Read multipart data and save in memory and in the disk if needed
 	form, err := reader.ReadForm(maxFormMemory)
 	if err != nil {
-		errorIf(err, "Unable to initialize multipart reader.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
 		return
 	}
@@ -507,9 +522,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	defer form.RemoveAll()
 
 	// Extract all form fields
-	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(form)
+	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
 	if err != nil {
-		errorIf(err, "Unable to parse form values.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
 		return
 	}
@@ -584,16 +599,16 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	// Extract metadata to be saved from received Form.
-	metadata, err := extractMetadataFromHeader(formValues)
+	metadata := make(map[string]string)
+	err = extractMetadataFromMap(ctx, formValues, metadata)
 	if err != nil {
-		errorIf(err, "found invalid http request header")
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
 
 	hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
 	if err != nil {
-		errorIf(err, "Unable to initialize hashReader.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -607,7 +622,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
-			reader, err = newEncryptReader(hashReader, key, metadata)
+			reader, err = newEncryptReader(hashReader, key, bucket, object, metadata)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -632,7 +647,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	w.Header().Set("Location", location)
 
 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -690,10 +705,11 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponseHeadersOnly(w, s3Error)
 		return
 	}
+
 	getBucketInfo := objectAPI.GetBucketInfo
 	if api.CacheAPI() != nil {
 		getBucketInfo = api.CacheAPI().GetBucketInfo
@@ -710,20 +726,20 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "DeleteBucket")
 
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
 
-	// DeleteBucket does not have any bucket action.
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
 	deleteBucket := objectAPI.DeleteBucket
 	if api.CacheAPI() != nil {
 		deleteBucket = api.CacheAPI().DeleteBucket
@@ -734,14 +750,17 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	// Notify all peers (including self) to update in-memory state
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
-		errorIf(err, "unable to update policy change in remote peer %v", addr)
-	}
-
 	globalNotificationSys.RemoveNotification(bucket)
-	for addr, err := range globalNotificationSys.DeleteBucket(bucket) {
-		errorIf(err, "unable to delete bucket in remote peer %v", addr)
+	globalPolicySys.Remove(bucket)
+	globalNotificationSys.DeleteBucket(ctx, bucket)
+
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			// Deleting DNS entry failed, attempt to create the bucket again.
+			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	// Write success response.

cmd/bucket-handlers_test.go

@@ -155,7 +155,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getReadOnlyBucketStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestGetBucketLocationHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyBucketStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestGetBucketLocationHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -260,7 +260,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getReadOnlyBucketStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestHeadBucketHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyBucketStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestHeadBucketHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -313,7 +313,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			maxUploads:         "0",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusBadRequest,
+			expectedRespStatus: http.StatusNotFound,
 			shouldPass:         false,
 		},
 		// Test case - 2.
@@ -494,7 +494,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyBucketStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestListMultipartUploadsHandler", bucketName, "", instanceType, apiRouter, anonReq, getWriteOnlyBucketStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestListMultipartUploadsHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -592,7 +592,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "ListBucketsHandler", "", "", instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "ListBucketsHandler", "", "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy("*"))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -793,31 +793,3 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	// `ExecObjectLayerAPINilTest` manages the operation.
 	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
 }
-
-func TestIsBucketActionAllowed(t *testing.T) {
-	ExecObjectLayerAPITest(t, testIsBucketActionAllowedHandler, []string{"BucketLocation"})
-}
-
-func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-
-	testCases := []struct {
-		// input.
-		action              string
-		bucket              string
-		prefix              string
-		isGlobalPoliciesNil bool
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		{"s3:GetBucketLocation", "mybucket", "abc", true, false},
-		{"s3:ListObject", "mybucket", "abc", false, false},
-	}
-	for i, testCase := range testCases {
-		isAllowed := isBucketActionAllowed(testCase.action, testCase.bucket, testCase.prefix, obj)
-		if isAllowed != testCase.shouldPass {
-			t.Errorf("Case %d: Expected the response status to be `%t`, but instead found `%t`", i+1, testCase.shouldPass, isAllowed)
-		}
-
-	}
-}

cmd/bucket-notification-handlers.go

@@ -23,10 +23,11 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	xerrors "github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
 	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/policy"
 )
 
 const (
@@ -43,6 +44,9 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "GetBucketNotification")
 
+	vars := mux.Vars(r)
+	bucketName := vars["bucket"]
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -53,27 +57,23 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
 		return
 	}
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucketName := vars["bucket"]
-
 	_, err := objAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Attempt to successfully load notification config.
-	nConfig, err := readNotificationConfig(objAPI, bucketName)
+	nConfig, err := readNotificationConfig(ctx, objAPI, bucketName)
 	if err != nil {
 		// Ignore errNoSuchNotifications to comply with AWS S3.
-		if xerrors.Cause(err) != errNoSuchNotifications {
-			errorIf(err, "Unable to read notification configuration.")
+		if err != errNoSuchNotifications {
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 			return
 		}
@@ -83,7 +83,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	notificationBytes, err := xml.Marshal(nConfig)
 	if err != nil {
-		errorIf(err, "Unable to marshal notification configuration into XML.", err)
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -106,14 +106,15 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
 		return
 	}
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -145,9 +146,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	rulesMap := config.ToRulesMap()
 	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
-	for addr, err := range globalNotificationSys.PutBucketNotification(bucketName, rulesMap) {
-		errorIf(err, "unable to put bucket notification to remote peer %v", addr)
-	}
+	globalNotificationSys.PutBucketNotification(ctx, bucketName, rulesMap)
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -167,14 +166,15 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
 		return
 	}
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	values := r.URL.Query()
 
 	var prefix string
@@ -217,39 +217,53 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	}
 
 	if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
-		errorIf(err, "Unable to get bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	host := xnet.MustParseHost(r.RemoteAddr)
-	target := target.NewHTTPClientTarget(*host, w)
+	host, err := xnet.ParseHost(r.RemoteAddr)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	target, err := target.NewHTTPClientTarget(*host, w)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
 	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
 
 	if err := globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
-		errorIf(err, "Unable to add httpclient target %v to globalNotificationSys.targetList.", target)
+		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
 	defer globalNotificationSys.RemoveRulesMap(bucketName, rulesMap)
 
-	thisAddr := xnet.MustParseHost(GetLocalPeer(globalEndpoints))
-	if err := SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
-		errorIf(err, "Unable to save HTTP listener %v", target)
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	errors := globalNotificationSys.ListenBucketNotification(bucketName, eventNames, pattern, target.ID(), *thisAddr)
-	for addr, err := range errors {
-		errorIf(err, "unable to call listen bucket notification to remote peer %v", addr)
+	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
+		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
+		logger.LogIf(ctx, err)
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
 	}
 
+	globalNotificationSys.ListenBucketNotification(ctx, bucketName, eventNames, pattern, target.ID(), *thisAddr)
+
 	<-target.DoneCh
 
-	if err := RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
-		errorIf(err, "Unable to save HTTP listener %v", target)
+	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
+		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}

cmd/bucket-policy-handlers.go

@@ -19,206 +19,24 @@ package cmd
 import (
 	"encoding/json"
 	"io"
-	"io/ioutil"
-	"net"
 	"net/http"
-	"runtime"
-	"strings"
 
 	humanize "github.com/dustin/go-humanize"
-	mux "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio/pkg/errors"
-	"github.com/minio/minio/pkg/wildcard"
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/policy"
 )
 
-// maximum supported access policy size.
-const maxAccessPolicySize = 20 * humanize.KiByte
+const (
+	// As per AWS S3 specification, 20KiB policy JSON data is allowed.
+	maxBucketPolicySize = 20 * humanize.KiByte
 
-// Verify if a given action is valid for the url path based on the
-// existing bucket access policy.
-func bucketPolicyEvalStatements(action string, resource string, conditions policy.ConditionKeyMap,
-	statements []policy.Statement) bool {
-	for _, statement := range statements {
-		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
-			if statement.Effect == "Allow" {
-				return true
-			}
-			// Do not uncomment kept here for readability.
-			// else statement.Effect == "Deny"
-			return false
-		}
-	}
-	// None match so deny.
-	return false
-}
+	// Policy configuration file.
+	bucketPolicyConfig = "policy.json"
+)
 
-// Verify if action, resource and conditions match input policy statement.
-func bucketPolicyMatchStatement(action string, resource string, conditions policy.ConditionKeyMap,
-	statement policy.Statement) bool {
-	// Verify if action, resource and condition match in given statement.
-	return (bucketPolicyActionMatch(action, statement) &&
-		bucketPolicyResourceMatch(resource, statement) &&
-		bucketPolicyConditionMatch(conditions, statement))
-}
-
-// Verify if given action matches with policy statement.
-func bucketPolicyActionMatch(action string, statement policy.Statement) bool {
-	return !statement.Actions.FuncMatch(actionMatch, action).IsEmpty()
-}
-
-// Match function matches wild cards in 'pattern' for resource.
-func resourceMatch(pattern, resource string) bool {
-	if runtime.GOOS == "windows" {
-		// For windows specifically make sure we are case insensitive.
-		return wildcard.Match(strings.ToLower(pattern), strings.ToLower(resource))
-	}
-	return wildcard.Match(pattern, resource)
-}
-
-// Match function matches wild cards in 'pattern' for action.
-func actionMatch(pattern, action string) bool {
-	return wildcard.MatchSimple(pattern, action)
-}
-
-func refererMatch(pattern, referer string) bool {
-	return wildcard.MatchSimple(pattern, referer)
-}
-
-// isIPInCIDR - checks if a given a IP address is a member of the given subnet.
-func isIPInCIDR(cidr, ip string) bool {
-	// AWS S3 spec says IPs must use standard CIDR notation.
-	// http://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html#example-bucket-policies-use-case-3.
-	_, cidrNet, err := net.ParseCIDR(cidr)
-	if err != nil {
-		return false // If provided CIDR can't be parsed no IP will be in the subnet.
-	}
-	addr := net.ParseIP(ip)
-	return cidrNet.Contains(addr)
-}
-
-// Verify if given resource matches with policy statement.
-func bucketPolicyResourceMatch(resource string, statement policy.Statement) bool {
-	// the resource rule for object could contain "*" wild card.
-	// the requested object can be given access based on the already set bucket policy if
-	// the match is successful.
-	// More info: http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html.
-	return !statement.Resources.FuncMatch(resourceMatch, resource).IsEmpty()
-}
-
-// Verify if given condition matches with policy statement.
-func bucketPolicyConditionMatch(conditions policy.ConditionKeyMap, statement policy.Statement) bool {
-	// Supports following conditions.
-	// - StringEquals
-	// - StringNotEquals
-	// - StringLike
-	// - StringNotLike
-	// - IpAddress
-	// - NotIpAddress
-	//
-	// Supported applicable condition keys for each conditions.
-	// - s3:prefix
-	// - s3:max-keys
-	// - s3:aws-Referer
-	// - s3:aws-SourceIp
-
-	// The following loop evaluates the logical AND of all the
-	// conditions in the statement. Note: we can break out of the
-	// loop if and only if a condition evaluates to false.
-	for condition, conditionKeyVal := range statement.Conditions {
-		prefixConditon := conditionKeyVal["s3:prefix"]
-		maxKeyCondition := conditionKeyVal["s3:max-keys"]
-		if condition == "StringEquals" {
-			// If there is no condition with "s3:prefix" or "s3:max-keys" condition key
-			// then there is nothing to check condition against.
-			if !prefixConditon.IsEmpty() && !prefixConditon.Equals(conditions["prefix"]) {
-				return false
-			}
-			if !maxKeyCondition.IsEmpty() && !maxKeyCondition.Equals(conditions["max-keys"]) {
-				return false
-			}
-		} else if condition == "StringNotEquals" {
-			// If there is no condition with "s3:prefix" or "s3:max-keys" condition key
-			// then there is nothing to check condition against.
-			if !prefixConditon.IsEmpty() && prefixConditon.Equals(conditions["prefix"]) {
-				return false
-			}
-			if !maxKeyCondition.IsEmpty() && maxKeyCondition.Equals(conditions["max-keys"]) {
-				return false
-			}
-		} else if condition == "StringLike" {
-			awsReferers := conditionKeyVal["aws:Referer"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsReferers.IsEmpty() {
-				continue
-			}
-			// wildcard match of referer in statement was not empty.
-			// StringLike has a match, i.e, condition evaluates to true.
-			refererFound := false
-			for referer := range conditions["referer"] {
-				if !awsReferers.FuncMatch(refererMatch, referer).IsEmpty() {
-					refererFound = true
-					break
-				}
-			}
-			// No matching referer found, so the condition is false.
-			if !refererFound {
-				return false
-			}
-		} else if condition == "StringNotLike" {
-			awsReferers := conditionKeyVal["aws:Referer"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsReferers.IsEmpty() {
-				continue
-			}
-			// wildcard match of referer in statement was not empty.
-			// StringNotLike has a match, i.e, condition evaluates to false.
-			for referer := range conditions["referer"] {
-				if !awsReferers.FuncMatch(refererMatch, referer).IsEmpty() {
-					return false
-				}
-			}
-		} else if condition == "IpAddress" {
-			awsIps := conditionKeyVal["aws:SourceIp"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsIps.IsEmpty() {
-				continue
-			}
-			// wildcard match of ip if statement was not empty.
-			// Find a valid ip.
-			ipFound := false
-			for ip := range conditions["ip"] {
-				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
-					ipFound = true
-					break
-				}
-			}
-			if !ipFound {
-				return false
-			}
-		} else if condition == "NotIpAddress" {
-			awsIps := conditionKeyVal["aws:SourceIp"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsIps.IsEmpty() {
-				continue
-			}
-			// wildcard match of ip if statement was not empty.
-			// Find if nothing matches.
-			for ip := range conditions["ip"] {
-				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
-					return false
-				}
-			}
-		}
-	}
-
-	return true
-}
-
-// PutBucketPolicyHandler - PUT Bucket policy
-// -----------------
-// This implementation of the PUT operation uses the policy
-// subresource to add to or replace a policy on a bucket
+// PutBucketPolicyHandler - This HTTP handler stores given bucket policy configuration as per
+// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "PutBucketPolicy")
 
@@ -228,77 +46,58 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	// If Content-Length is unknown or zero, deny the
-	// request. PutBucketPolicy always needs a Content-Length.
-	if r.ContentLength == -1 || r.ContentLength == 0 {
+	// Error out if Content-Length is missing.
+	// PutBucketPolicy always needs Content-Length.
+	if r.ContentLength <= 0 {
 		writeErrorResponse(w, ErrMissingContentLength, r.URL)
 		return
 	}
-	// If Content-Length is greater than maximum allowed policy size.
-	if r.ContentLength > maxAccessPolicySize {
+
+	// Error out if Content-Length is beyond allowed size.
+	if r.ContentLength > maxBucketPolicySize {
 		writeErrorResponse(w, ErrEntityTooLarge, r.URL)
 		return
 	}
 
-	// Read access policy up to maxAccessPolicySize.
-	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
-	// bucket policies are limited to 20KB in size, using a limit reader.
-	policyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		errorIf(err, "Unable to read from client.")
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	// Version in policy must not be empty
+	if bucketPolicy.Version == "" {
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
-	policyInfo := policy.BucketAccessPolicy{}
-	if err = json.Unmarshal(policyBytes, &policyInfo); err != nil {
-		writeErrorResponse(w, ErrInvalidPolicyDocument, r.URL)
-		return
-	}
-	// Parse check bucket policy.
-	if s3Error := checkBucketPolicyResources(bucket, policyInfo); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
 
-	if err = objAPI.SetBucketPolicy(ctx, bucket, policyInfo); err != nil {
-		err = errors.Cause(err)
-		switch err.(type) {
-		case NotImplemented:
-			// Return error for invalid bucket name.
-			writeErrorResponse(w, ErrNotImplemented, r.URL)
-		default:
-			writeErrorResponse(w, ErrMalformedPolicy, r.URL)
-		}
-		return
-	}
-
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
-		errorIf(err, "unable to update policy change in remote peer %v", addr)
-	}
+	globalPolicySys.Set(bucket, *bucketPolicy)
+	globalNotificationSys.SetBucketPolicy(ctx, bucket, bucketPolicy)
 
 	// Success.
 	writeSuccessNoContent(w)
 }
 
-// DeleteBucketPolicyHandler - DELETE Bucket policy
-// -----------------
-// This implementation of the DELETE operation uses the policy
-// subresource to add to remove a policy on a bucket.
+// DeleteBucketPolicyHandler - This HTTP handler removes bucket policy configuration.
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "DeleteBucketPolicy")
 
@@ -308,40 +107,33 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	// Delete bucket access policy, by passing an empty policy
-	// struct.
 	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
-		errorIf(err, "unable to update policy change in remote peer %v", addr)
-	}
+	globalPolicySys.Remove(bucket)
+	globalNotificationSys.RemoveBucketPolicy(ctx, bucket)
 
 	// Success.
 	writeSuccessNoContent(w)
 }
 
-// GetBucketPolicyHandler - GET Bucket policy
-// -----------------
-// This operation uses the policy
-// subresource to return the policy of a specified bucket.
+// GetBucketPolicyHandler - This HTTP handler returns bucket policy configuration.
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "GetBucketPolicy")
 
@@ -351,35 +143,34 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Read bucket access policy.
-	policy, err := objAPI.GetBucketPolicy(ctx, bucket)
+	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	policyBytes, err := json.Marshal(&policy)
+	policyData, err := json.Marshal(bucketPolicy)
 	if err != nil {
-		errorIf(err, "Unable to marshal bucket policy.")
+		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Write to client.
-	w.Write(policyBytes)
+	w.Write(policyData)
 }

cmd/bucket-policy-handlers_test.go

@@ -24,222 +24,72 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
+	"strings"
 	"testing"
 
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/policy/condition"
 )
 
-// Tests validate Bucket policy resource matcher.
-func TestBucketPolicyResourceMatch(t *testing.T) {
-
-	// generates statement with given resource..
-	generateStatement := func(resource string) policy.Statement {
-		statement := policy.Statement{}
-		statement.Resources = set.CreateStringSet([]string{resource}...)
-		return statement
-	}
-
-	// generates resource prefix.
-	generateResource := func(bucketName, objectName string) string {
-		return bucketARNPrefix + bucketName + "/" + objectName
-	}
-
-	testCases := []struct {
-		resourceToMatch       string
-		statement             policy.Statement
-		expectedResourceMatch bool
-	}{
-		// Test case 1-4.
-		// Policy with resource ending with bucket/* allows access to all objects inside the given bucket.
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		// Test case - 5.
-		// Policy with resource ending with bucket/oo* should not allow access to bucket/output.txt.
-		{generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), false},
-		// Test case - 6.
-		// Policy with resource ending with bucket/oo* should allow access to bucket/ootput.txt.
-		{generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), true},
-		// Test case - 7.
-		// Policy with resource ending with bucket/oo* allows access to all sub-dirs starting with "oo" inside given bucket.
-		{generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), true},
-		// Test case - 8.
-		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/Asia/Japan/*")), false},
-		// Test case - 9.
-		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/Asia/Japan/*")), false},
-		// Test case - 10.
-		// Proves that the name space is flat.
-		{generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix,
-			"minio-bucket"+"/*/India/*/Bihar")), true},
-		// Test case - 11.
-		// Proves that the name space is flat.
-		{generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix,
-			"minio-bucket"+"/*/India/*/Bihar/*")), true},
-	}
-	for i, testCase := range testCases {
-		actualResourceMatch := bucketPolicyResourceMatch(testCase.resourceToMatch, testCase.statement)
-		if testCase.expectedResourceMatch != actualResourceMatch {
-			t.Errorf("Test %d: Expected Resource match to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResourceMatch, actualResourceMatch)
-		}
+func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
+			policy.NewResourceSet(policy.NewResource(bucketName, "")),
+			condition.NewFunctions(),
+		)},
 	}
 }
 
-// TestBucketPolicyActionMatch - Test validates whether given action on the
-// bucket/object matches the allowed actions in policy.Statement.
-// This test preserves the allowed actions for all 3 sets of policies, that is read-write,read-only, write-only.
-// The intention of the test is to catch any changes made to allowed action for on eof the above 3 major policy groups mentioned.
-func TestBucketPolicyActionMatch(t *testing.T) {
-	bucketName := getRandomBucketName()
-	objectPrefix := "test-object"
-
-	testCases := []struct {
-		action         string
-		statement      policy.Statement
-		expectedResult bool
-	}{
-		// s3:GetBucketLocation is the action necessary to be present in the bucket policy to allow
-		// fetching of bucket location on an Anonymous/unsigned request.
-
-		//r ead-write bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 1).
-		{"s3:GetBucketLocation", getReadWriteBucketStatement(bucketName, objectPrefix), true},
-		//	write-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 2).
-		{"s3:GetBucketLocation", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
-		//	read-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 3).
-		{"s3:GetBucketLocation", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
-
-		// Any of the Object level access permissions shouldn't allow for GetBucketLocation operation  on an Anonymous/unsigned request (Test cases 4-6).
-		{"s3:GetBucketLocation", getReadWriteObjectStatement(bucketName, objectPrefix), false},
-		{"s3:GetBucketLocation", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-		{"s3:GetBucketLocation", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:ListBucketMultipartUploads is the action necessary to be present in the bucket policy to allow
-		// Listing of multipart uploads in a given bucket for an Anonymous/unsigned request.
-
-		//read-write bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 7).
-		{"s3:ListBucketMultipartUploads", getReadWriteBucketStatement(bucketName, objectPrefix), true},
-		//	write-only bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 8).
-		{"s3:ListBucketMultipartUploads", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
-		// read-only bucket policy is expected to not allow ListBucketMultipartUploads operation on an anonymous request (Test case 9).
-		// the allowed actions in read-only bucket statement are  "s3:GetBucketLocation","s3:ListBucket",
-		// this should not allow for ListBucketMultipartUploads operations.
-		{"s3:ListBucketMultipartUploads", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Any of the object level policy will not allow for s3:ListBucketMultipartUploads (Test cases 10-12).
-		{"s3:ListBucketMultipartUploads", getReadWriteObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucketMultipartUploads", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucketMultipartUploads", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:ListBucket is the action necessary to be present in the bucket policy to allow
-		// listing of all objects inside a given bucket on an Anonymous/unsigned request.
-
-		// Cases for testing ListBucket access for different Bucket level access permissions.
-		// read-only bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 13).
-		{"s3:ListBucket", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
-		// read-write bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 14).
-		{"s3:ListBucket", getReadWriteBucketStatement(bucketName, objectPrefix), true},
-		// write-only  bucket policy is expected to not allow ListBucket operation on an anonymous request (Test case 15).
-		// the allowed actions in write-only  bucket statement are "s3:GetBucketLocation",	"s3:ListBucketMultipartUploads",
-		// this should not allow for ListBucket operations.
-		{"s3:ListBucket", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing ListBucket access for different Object level access permissions (Test cases 16-18).
-		// Any of the Object level access permissions shouldn't allow for ListBucket operation  on an Anonymous/unsigned request.
-		{"s3:ListBucket", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucket", getReadWriteObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucket", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:DeleteObject is the action necessary to be present in the bucket policy to allow
-		// deleting/removal of  objects inside a given bucket for an Anonymous/unsigned request.
-
-		// Cases for testing DeleteObject access for different Bucket level access permissions (Test cases 19-21).
-		// Any of the Bucket level access permissions shouldn't allow for DeleteObject operation  on an Anonymous/unsigned request.
-		{"s3:DeleteObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:DeleteObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:DeleteObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing DeleteObject access for different Object level access permissions (Test cases 22).
-		// read-only bucket policy is expected to not allow Delete Object operation on an anonymous request.
-		{"s3:DeleteObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write bucket policy is expected to allow Delete Bucket operation on an anonymous request (Test cases 23).
-		{"s3:DeleteObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  bucket policy is expected to allow Delete Object operation on an anonymous request (Test cases 24).
-		{"s3:DeleteObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
-
-		// s3:AbortMultipartUpload is the action necessary to be present in the bucket policy to allow
-		// cancelling or abortion of an already initiated multipart upload operation for an Anonymous/unsigned request.
-
-		// Cases for testing AbortMultipartUpload access for different Bucket level access permissions (Test cases 25-27).
-		// Any of the Bucket level access permissions shouldn't allow for AbortMultipartUpload operation on an Anonymous/unsigned request.
-		{"s3:AbortMultipartUpload", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:AbortMultipartUpload", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:AbortMultipartUpload", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing AbortMultipartUpload access for different Object level access permissions.
-		// read-only object policy is expected to not allow AbortMultipartUpload operation on an anonymous request (Test case 28).
-		{"s3:AbortMultipartUpload", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 29).
-		{"s3:AbortMultipartUpload", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 30).
-		{"s3:AbortMultipartUpload", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
-
-		// s3:PutObject is the action necessary to be present in the bucket policy to allow
-		// uploading of an object for an Anonymous/unsigned request.
-
-		// Cases for testing PutObject access for different Bucket level access permissions (Test cases 31-33).
-		// Any of the Bucket level access permissions shouldn't allow for PutObject operation on an Anonymous/unsigned request.
-		{"s3:PutObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:PutObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:PutObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing PutObject access for different Object level access permissions.
-		// read-only object policy is expected to not allow PutObject operation on an anonymous request (Test case 34).
-		{"s3:PutObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write object policy is expected to allow PutObject operation on an anonymous request (Test case 35).
-		{"s3:PutObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  object policy is expected to allow PutObject operation on an anonymous request (Test case 36).
-		{"s3:PutObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
-
-		// s3:GetObject is the action necessary to be present in the bucket policy to allow
-		// downloading of an object for an Anonymous/unsigned request.
-
-		// Cases for testing GetObject access for different Bucket level access permissions (Test cases 37-39).
-		// Any of the Bucket level access permissions shouldn't allow for GetObject operation on an Anonymous/unsigned request.
-		{"s3:GetObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:GetObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:GetObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing GetObject access for different Object level access permissions.
-		// read-only bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 40).
-		{"s3:GetObject", getReadOnlyObjectStatement(bucketName, objectPrefix), true},
-		// read-write bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 41).
-		{"s3:GetObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  bucket policy is expected to not allow  downloading of an Object on an anonymous request (Test case 42).
-		{"s3:GetObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:ListMultipartUploadParts is the action necessary to be present in the bucket policy to allow
-		// Listing of uploaded parts for an Anonymous/unsigned request.
-
-		// Any of the Bucket level access permissions shouldn't allow for ListMultipartUploadParts operation on an Anonymous/unsigned request.
-		// read-only bucket policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test cases 43-45).
-		{"s3:ListMultipartUploadParts", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:ListMultipartUploadParts", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:ListMultipartUploadParts", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// read-only object policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test case 46).
-		{"s3:ListMultipartUploadParts", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 47).
-		{"s3:ListMultipartUploadParts", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 48).
-		{"s3:ListMultipartUploadParts", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(
+				policy.GetBucketLocationAction,
+				policy.ListBucketMultipartUploadsAction,
+			),
+			policy.NewResourceSet(policy.NewResource(bucketName, "")),
+			condition.NewFunctions(),
+		)},
 	}
-	for i, testCase := range testCases {
-		actualResult := bucketPolicyActionMatch(testCase.action, testCase.statement)
-		if testCase.expectedResult != actualResult {
-			t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResult, actualResult)
-		}
+}
+
+func getAnonReadOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(policy.GetObjectAction),
+			policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+			condition.NewFunctions(),
+		)},
+	}
+}
+
+func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(
+				policy.AbortMultipartUploadAction,
+				policy.DeleteObjectAction,
+				policy.ListMultipartUploadPartsAction,
+				policy.PutObjectAction,
+			),
+			policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+			condition.NewFunctions(),
+		)},
 	}
 }
 
@@ -260,6 +110,8 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// template for constructing HTTP request body for PUT bucket policy.
 	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}`
 
+	bucketPolicyTemplateWithoutVersion := `{"Version":"","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}`
+
 	// test cases with sample input and expected output.
 	testCases := []struct {
 		bucketName string
@@ -290,7 +142,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			bucketName:         bucketName,
 			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
 
-			policyLen:          maxAccessPolicySize + 1,
+			policyLen:          maxBucketPolicySize + 1,
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			expectedRespStatus: http.StatusBadRequest,
@@ -357,7 +209,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		// Test case - 8.
 		// non-existent bucket is used.
 		// writing BucketPolicy should fail.
-		// should result is 500 InternalServerError.
+		// should result in 404 StatusNotFound
 		{
 			bucketName:         "non-existent-bucket",
 			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, "non-existent-bucket", "non-existent-bucket"))),
@@ -368,9 +220,9 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 9.
-		// invalid bucket name is used.
+		// non-existent bucket is used (with invalid bucket name)
 		// writing BucketPolicy should fail.
-		// should result is 400 StatusBadRequest.
+		// should result in 404 StatusNotFound
 		{
 			bucketName:         ".invalid-bucket",
 			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, ".invalid-bucket", ".invalid-bucket"))),
@@ -378,6 +230,19 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			policyLen:          len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
+			expectedRespStatus: http.StatusNotFound,
+		},
+		// Test case - 10.
+		// Existent bucket with policy with Version field empty.
+		// writing BucketPolicy should fail.
+		// should result in 400 StatusBadRequest.
+		{
+			bucketName:         bucketName,
+			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplateWithoutVersion, bucketName, bucketName))),
+
+			policyLen:          len(fmt.Sprintf(bucketPolicyTemplateWithoutVersion, bucketName, bucketName)),
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
 			expectedRespStatus: http.StatusBadRequest,
 		},
 	}
@@ -430,7 +295,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "PutBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "PutBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -459,7 +324,7 @@ func TestGetBucketPolicyHandler(t *testing.T) {
 func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
 	// template for constructing HTTP request body for PUT bucket policy.
-	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"],"Sid":""}]}`
+	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"]},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"]}]}`
 
 	// Writing bucket policy before running test on GetBucketPolicy.
 	putTestPolicies := []struct {
@@ -535,13 +400,13 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			expectedRespStatus:   http.StatusNotFound,
 		},
 		// Test case - 3.
-		// Case with invalid bucket name.
+		// Case with non-existent bucket name.
 		{
 			bucketName:           ".invalid-bucket-name",
 			accessKey:            credentials.AccessKey,
 			secretKey:            credentials.SecretKey,
 			expectedBucketPolicy: "",
-			expectedRespStatus:   http.StatusBadRequest,
+			expectedRespStatus:   http.StatusNotFound,
 		},
 	}
 	// Iterating over the cases, fetching the policy and validating the response.
@@ -572,7 +437,16 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 		if recV4.Code != testCase.expectedRespStatus {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			if expectedBucketPolicyStr != string(bucketPolicyReadBuf) {
+			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+
+			if !reflect.DeepEqual(expectedPolicy, gotPolicy) {
 				t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType)
 			}
 		}
@@ -598,7 +472,16 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		}
 		if recV2.Code == http.StatusOK {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			if expectedBucketPolicyStr != string(bucketPolicyReadBuf) {
+			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+
+			if !reflect.DeepEqual(expectedPolicy, gotPolicy) {
 				t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType)
 			}
 		}
@@ -617,7 +500,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "GetBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "GetBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -742,12 +625,12 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 3.
-		// Case with invalid bucket name.
+		// Case with non-existent-bucket.
 		{
 			bucketName:         ".invalid-bucket-name",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusBadRequest,
+			expectedRespStatus: http.StatusNotFound,
 		},
 	}
 	// Iterating over the cases and deleting the bucket policy and then asserting response.
@@ -820,7 +703,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "DeleteBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "DeleteBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -838,174 +721,3 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	// `ExecObjectLayerAPINilTest` manages the operation.
 	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
 }
-
-// TestBucketPolicyConditionMatch - Tests to validate whether bucket policy conditions match.
-func TestBucketPolicyConditionMatch(t *testing.T) {
-	// obtain the inner map[string]set.StringSet for policy.Statement.Conditions.
-	getInnerMap := func(key2, value string) map[string]set.StringSet {
-		innerMap := make(map[string]set.StringSet)
-		innerMap[key2] = set.CreateStringSet(value)
-		return innerMap
-	}
-
-	// obtain policy.Statement with Conditions set.
-	getStatementWithCondition := func(key1, key2, value string) policy.Statement {
-		innerMap := getInnerMap(key2, value)
-		// to set policyStatment.Conditions .
-		conditions := make(policy.ConditionMap)
-		conditions[key1] = innerMap
-		// new policy statement.
-		statement := policy.Statement{}
-		// set the condition.
-		statement.Conditions = conditions
-		return statement
-	}
-
-	testCases := []struct {
-		statementCondition policy.Statement
-		condition          map[string]set.StringSet
-
-		expectedMatch bool
-	}{
-
-		// Test case - 1.
-		// StringEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Asia/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 2.
-		// StringEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Africa/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 3.
-		// StringEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Asia/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 4.
-		// StringEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Africa/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 5.
-		// StringNotEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Asia/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 6.
-		// StringNotEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Africa/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 7.
-		// StringNotEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Asia/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 8.
-		// StringNotEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Africa/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 9.
-		// StringLike condition matches.
-		{
-			statementCondition: getStatementWithCondition("StringLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "http://www.example.com/"),
-			expectedMatch:      true,
-		},
-		// Test case - 10.
-		// StringLike condition doesn't match.
-		{
-			statementCondition: getStatementWithCondition("StringLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "www.somethingelse.com"),
-			expectedMatch:      false,
-		},
-		// Test case - 11.
-		// StringNotLike condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("StringNotLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "http://www.example.com/"),
-			expectedMatch:      false,
-		},
-		// Test case - 12.
-		// StringNotLike condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("StringNotLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "http://somethingelse.com/"),
-			expectedMatch:      true,
-		},
-		// Test case 13.
-		// IpAddress condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.143.2"),
-			expectedMatch:      true,
-		},
-		// Test case 14.
-		// IpAddress condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "127.240.143.224"),
-			expectedMatch:      false,
-		},
-		// Test case 15.
-		// NotIpAddress condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.144.188"),
-			expectedMatch:      true,
-		},
-		// Test case 16.
-		// NotIpAddress condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.143.243"),
-			expectedMatch:      false,
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("Case %d", i+1), func(t *testing.T) {
-			// call the function under test and assert the result with the expected result.
-			doesMatch := bucketPolicyConditionMatch(tc.condition, tc.statementCondition)
-			if tc.expectedMatch != doesMatch {
-				t.Errorf("Expected the match to be `%v`; got `%v` - %v %v.",
-					tc.expectedMatch, doesMatch, tc.condition, tc.statementCondition)
-			}
-		})
-	}
-}

cmd/bucket-policy-parser.go

@@ -1,295 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Package cmd This file implements AWS Access Policy Language parser in
-// accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
-package cmd
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"sort"
-	"strings"
-
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
-)
-
-var emptyBucketPolicy = policy.BucketAccessPolicy{}
-
-var conditionKeyActionMap = policy.ConditionKeyMap{
-	"s3:prefix": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads"),
-	"s3:max-keys": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads",
-		"s3:ListMultipartUploadParts"),
-}
-
-// supportedActionMap - lists all the actions supported by minio.
-var supportedActionMap = set.CreateStringSet("*", "s3:*", "s3:GetObject",
-	"s3:ListBucket", "s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
-	"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts")
-
-// supported Conditions type.
-var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals", "StringLike", "StringNotLike", "IpAddress", "NotIpAddress")
-
-// Validate s3:prefix, s3:max-keys are present if not
-// supported keys for the conditions.
-var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aws:Referer", "aws:SourceIp")
-
-// supportedEffectMap - supported effects.
-var supportedEffectMap = set.CreateStringSet("Allow", "Deny")
-
-// isValidActions - are actions valid.
-func isValidActions(actions set.StringSet) (err error) {
-	// Statement actions cannot be empty.
-	if actions.IsEmpty() {
-		err = errors.New("Action list cannot be empty")
-		return err
-	}
-	if unsupportedActions := actions.Difference(supportedActionMap); !unsupportedActions.IsEmpty() {
-		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document",
-			unsupportedActions)
-		return err
-	}
-	return nil
-}
-
-// isValidEffect - is effect valid.
-func isValidEffect(effect string) (err error) {
-	// Statement effect cannot be empty.
-	if effect == "" {
-		err = errors.New("Policy effect cannot be empty")
-		return err
-	}
-	if !supportedEffectMap.Contains(effect) {
-		err = errors.New("Unsupported Effect found: ‘" + effect + "’, please validate your policy document")
-		return err
-	}
-	return nil
-}
-
-// isValidResources - are valid resources.
-func isValidResources(resources set.StringSet) (err error) {
-	// Statement resources cannot be empty.
-	if resources.IsEmpty() {
-		err = errors.New("Resource list cannot be empty")
-		return err
-	}
-	for resource := range resources {
-		if !hasPrefix(resource, bucketARNPrefix) {
-			err = errors.New("Unsupported resource style found: ‘" + resource + "’, please validate your policy document")
-			return err
-		}
-		resourceSuffix := strings.SplitAfter(resource, bucketARNPrefix)[1]
-		if len(resourceSuffix) == 0 || hasPrefix(resourceSuffix, "/") {
-			err = errors.New("Invalid resource style found: ‘" + resource + "’, please validate your policy document")
-			return err
-		}
-	}
-	return nil
-}
-
-// isValidPrincipals - are valid principals.
-func isValidPrincipals(principal policy.User) (err error) {
-	if principal.AWS.IsEmpty() {
-		return errors.New("Principal cannot be empty")
-	}
-	if diff := principal.AWS.Difference(set.CreateStringSet("*")); !diff.IsEmpty() {
-		// Minio does not support or implement IAM, "*" is the only valid value.
-		// Amazon s3 doc on principal:
-		// http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
-		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document",
-			diff)
-		return err
-	}
-	return nil
-}
-
-// isValidConditions - returns nil if the given conditions valid and
-// corresponding error otherwise.
-func isValidConditions(actions set.StringSet, conditions policy.ConditionMap) (err error) {
-	// Verify conditions should be valid. Validate if only
-	// supported condition keys are present and return error
-	// otherwise.
-	conditionKeyVal := make(map[string]set.StringSet)
-	for conditionType := range conditions {
-		if !supportedConditionsType.Contains(conditionType) {
-			err = fmt.Errorf("Unsupported condition type '%s', please validate your policy document", conditionType)
-			return err
-		}
-		for key, value := range conditions[conditionType] {
-			if !supportedConditionsKey.Contains(key) {
-				err = fmt.Errorf("Unsupported condition key '%s', please validate your policy document", conditionType)
-				return err
-			}
-
-			compatibleActions := conditionKeyActionMap[key]
-			if !compatibleActions.IsEmpty() &&
-				compatibleActions.Intersection(actions).IsEmpty() {
-				err = fmt.Errorf("Unsupported condition key %s for the given actions %s, "+
-					"please validate your policy document", key, actions)
-				return err
-			}
-
-			conditionVal, ok := conditionKeyVal[key]
-			if ok && !value.Intersection(conditionVal).IsEmpty() {
-				err = fmt.Errorf("Ambigious condition values for key '%s', please validate your policy document", key)
-				return err
-			}
-			conditionKeyVal[key] = value
-		}
-	}
-	return nil
-}
-
-// List of actions for which prefixes are not allowed.
-var invalidPrefixActions = set.StringSet{
-	"s3:GetBucketLocation":          {},
-	"s3:ListBucket":                 {},
-	"s3:ListBucketMultipartUploads": {},
-	// Add actions which do not honor prefixes.
-}
-
-// resourcePrefix - provides the prefix removing any wildcards.
-func resourcePrefix(resource string) string {
-	if strings.HasSuffix(resource, "*") {
-		resource = strings.TrimSuffix(resource, "*")
-	}
-	return resource
-}
-
-// checkBucketPolicyResources validates Resources in unmarshalled bucket policy structure.
-// - Resources are validated against the given set of Actions.
-// -
-func checkBucketPolicyResources(bucket string, bucketPolicy policy.BucketAccessPolicy) APIErrorCode {
-	// Validate statements for special actions and collect resources
-	// for others to validate nesting.
-	var resourceMap = set.NewStringSet()
-	for _, statement := range bucketPolicy.Statements {
-		for action := range statement.Actions {
-			for resource := range statement.Resources {
-				resourcePrefix := strings.SplitAfter(resource, bucketARNPrefix)[1]
-				if _, ok := invalidPrefixActions[action]; ok {
-					// Resource prefix is not equal to bucket for
-					// prefix invalid actions, reject them.
-					if resourcePrefix != bucket {
-						return ErrMalformedPolicy
-					}
-				} else {
-					// For all other actions validate if resourcePrefix begins
-					// with bucket name, if not reject them.
-					if strings.Split(resourcePrefix, "/")[0] != bucket {
-						return ErrMalformedPolicy
-					}
-					// All valid resources collect them separately to verify nesting.
-					resourceMap.Add(resourcePrefix)
-				}
-			}
-		}
-	}
-
-	var resources []string
-	for resource := range resourceMap {
-		resources = append(resources, resourcePrefix(resource))
-	}
-
-	// Sort strings as shorter first.
-	sort.Strings(resources)
-
-	for len(resources) > 1 {
-		var resource string
-		resource, resources = resources[0], resources[1:]
-		// Loop through all resources, if one of them matches with
-		// previous shorter one, it means we have detected
-		// nesting. Reject such rules.
-		for _, otherResource := range resources {
-			// Common prefix reject such rules.
-			if hasPrefix(otherResource, resource) {
-				return ErrPolicyNesting
-			}
-		}
-	}
-
-	// No errors found.
-	return ErrNone
-}
-
-// parseBucketPolicy - parses and validates if bucket policy is of
-// proper JSON and follows allowed restrictions with policy standards.
-func parseBucketPolicy(bucketPolicyReader io.Reader, bktPolicy *policy.BucketAccessPolicy) (err error) {
-	// Parse bucket policy reader.
-	decoder := json.NewDecoder(bucketPolicyReader)
-	if err = decoder.Decode(bktPolicy); err != nil {
-		return err
-	}
-
-	// Policy version cannot be empty.
-	if len(bktPolicy.Version) == 0 {
-		err = errors.New("Policy version cannot be empty")
-		return err
-	}
-
-	// Policy statements cannot be empty.
-	if len(bktPolicy.Statements) == 0 {
-		err = errors.New("Policy statement cannot be empty")
-		return err
-	}
-
-	// Loop through all policy statements and validate entries.
-	for _, statement := range bktPolicy.Statements {
-		// Statement effect should be valid.
-		if err := isValidEffect(statement.Effect); err != nil {
-			return err
-		}
-		// Statement principal should be supported format.
-		if err := isValidPrincipals(statement.Principal); err != nil {
-			return err
-		}
-		// Statement actions should be valid.
-		if err := isValidActions(statement.Actions); err != nil {
-			return err
-		}
-		// Statement resources should be valid.
-		if err := isValidResources(statement.Resources); err != nil {
-			return err
-		}
-		// Statement conditions should be valid.
-		if err := isValidConditions(statement.Actions, statement.Conditions); err != nil {
-			return err
-		}
-	}
-
-	// Separate deny and allow statements, so that we can apply deny
-	// statements in the beginning followed by Allow statements.
-	var denyStatements []policy.Statement
-	var allowStatements []policy.Statement
-	for _, statement := range bktPolicy.Statements {
-		if statement.Effect == "Deny" {
-			denyStatements = append(denyStatements, statement)
-			continue
-		}
-
-		// else if statement.Effect == "Allow"
-		allowStatements = append(allowStatements, statement)
-	}
-
-	// Deny statements are enforced first once matched.
-	bktPolicy.Statements = append(denyStatements, allowStatements...)
-
-	// Return successfully parsed policy structure.
-	return nil
-}

cmd/bucket-policy-parser_test.go

@@ -1,867 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package cmd
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"reflect"
-	"testing"
-
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
-)
-
-// Common bucket actions for both read and write policies.
-var (
-	readWriteBucketActions = []string{
-		"s3:GetBucketLocation",
-		"s3:ListBucket",
-		"s3:ListBucketMultipartUploads",
-		// Add more bucket level read-write actions here.
-	}
-	readWriteObjectActions = []string{
-		"s3:AbortMultipartUpload",
-		"s3:DeleteObject",
-		"s3:GetObject",
-		"s3:ListMultipartUploadParts",
-		"s3:PutObject",
-		// Add more object level read-write actions here.
-	}
-)
-
-// Write only actions.
-var (
-	writeOnlyBucketActions = []string{
-		"s3:GetBucketLocation",
-		"s3:ListBucketMultipartUploads",
-		// Add more bucket level write actions here.
-	}
-	writeOnlyObjectActions = []string{
-		"s3:AbortMultipartUpload",
-		"s3:DeleteObject",
-		"s3:ListMultipartUploadParts",
-		"s3:PutObject",
-		// Add more object level write actions here.
-	}
-)
-
-// Read only actions.
-var (
-	readOnlyBucketActions = []string{
-		"s3:GetBucketLocation",
-		"s3:ListBucket",
-		// Add more bucket level read actions here.
-	}
-	readOnlyObjectActions = []string{
-		"s3:GetObject",
-		// Add more object level read actions here.
-	}
-)
-
-// Obtain bucket statement for read-write bucketPolicy.
-func getReadWriteObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
-	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
-	objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...)
-	return objectResourceStatement
-}
-
-// Obtain object statement for read-write bucketPolicy.
-func getReadWriteBucketStatement(bucketName, objectPrefix string) policy.Statement {
-	bucketResourceStatement := policy.Statement{}
-	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
-	bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...)
-	return bucketResourceStatement
-}
-
-// Obtain statements for read-write bucketPolicy.
-func getReadWriteStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
-	// Save the read write policy.
-	statements = append(statements, getReadWriteBucketStatement(bucketName, objectPrefix), getReadWriteObjectStatement(bucketName, objectPrefix))
-	return statements
-}
-
-// Obtain bucket statement for read only bucketPolicy.
-func getReadOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
-	bucketResourceStatement := policy.Statement{}
-	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
-	bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...)
-	return bucketResourceStatement
-}
-
-// Obtain object statement for read only bucketPolicy.
-func getReadOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
-	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
-	objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...)
-	return objectResourceStatement
-}
-
-// Obtain statements for read only bucketPolicy.
-func getReadOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
-	// Save the read only policy.
-	statements = append(statements, getReadOnlyBucketStatement(bucketName, objectPrefix), getReadOnlyObjectStatement(bucketName, objectPrefix))
-	return statements
-}
-
-// Obtain bucket statements for write only bucketPolicy.
-func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
-
-	bucketResourceStatement := policy.Statement{}
-	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
-	bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...)
-	return bucketResourceStatement
-}
-
-// Obtain object statements for write only bucketPolicy.
-func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
-	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
-	objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...)
-	return objectResourceStatement
-}
-
-// Obtain statements for write only bucketPolicy.
-func getWriteOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
-	// Write only policy.
-	// Save the write only policy.
-	statements = append(statements, getWriteOnlyBucketStatement(bucketName, objectPrefix), getWriteOnlyBucketStatement(bucketName, objectPrefix))
-	return statements
-}
-
-// Tests validate Action validator.
-func TestIsValidActions(t *testing.T) {
-	testCases := []struct {
-		// input.
-		actions set.StringSet
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Action.
-		// Test case - 1.
-		// "s3:ListObject" is an invalid Action.
-		{set.CreateStringSet([]string{"s3:GetObject", "s3:ListObject", "s3:RemoveObject"}...),
-			errors.New("Unsupported actions found: ‘set.StringSet{\"s3:RemoveObject\":struct {}{}, \"s3:ListObject\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 2.
-		// Empty Actions.
-		{set.CreateStringSet([]string{}...), errors.New("Action list cannot be empty"), false},
-		// Test case - 3.
-		// "s3:DeleteEverything"" is an invalid Action.
-		{set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...),
-			errors.New("Unsupported actions found: ‘set.StringSet{\"s3:DeleteEverything\":struct {}{}}’, please validate your policy document"), false},
-		// Inputs with valid Action.
-		// Test Case - 4.
-		{set.CreateStringSet([]string{
-			"s3:*", "*", "s3:GetObject", "s3:ListBucket",
-			"s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
-			"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads",
-			"s3:ListMultipartUploadParts"}...), nil, true},
-	}
-	for i, testCase := range testCases {
-		err := isValidActions(testCase.actions)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-	}
-}
-
-// Tests validate Effect validator.
-func TestIsValidEffect(t *testing.T) {
-	testCases := []struct {
-		// input.
-		effect string
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Effect.
-		// Test case - 1.
-		{"", errors.New("Policy effect cannot be empty"), false},
-		// Test case - 2.
-		{"DontAllow", errors.New("Unsupported Effect found: ‘DontAllow’, please validate your policy document"), false},
-		// Test case - 3.
-		{"NeverAllow", errors.New("Unsupported Effect found: ‘NeverAllow’, please validate your policy document"), false},
-		// Test case - 4.
-		{"AllowAlways", errors.New("Unsupported Effect found: ‘AllowAlways’, please validate your policy document"), false},
-
-		// Inputs with valid Effect.
-		// Test Case - 5.
-		{"Allow", nil, true},
-		// Test Case - 6.
-		{"Deny", nil, true},
-	}
-	for i, testCase := range testCases {
-		err := isValidEffect(testCase.effect)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-	}
-}
-
-// Tests validate Resources validator.
-func TestIsValidResources(t *testing.T) {
-	testCases := []struct {
-		// input.
-		resources []string
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Action.
-		// Test case - 1.
-		// Empty Resources.
-		{[]string{}, errors.New("Resource list cannot be empty"), false},
-		// Test case - 2.
-		// A valid resource should have prefix bucketARNPrefix.
-		{[]string{"my-resource"}, errors.New("Unsupported resource style found: ‘my-resource’, please validate your policy document"), false},
-		// Test case - 3.
-		// A Valid resource should have bucket name followed by bucketARNPrefix.
-		{[]string{bucketARNPrefix}, errors.New("Invalid resource style found: ‘arn:aws:s3:::’, please validate your policy document"), false},
-		// Test Case - 4.
-		// Valid resource shouldn't have slash('/') followed by bucketARNPrefix.
-		{[]string{bucketARNPrefix + "/"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::/’, please validate your policy document"), false},
-
-		// Test cases with valid Resources.
-		{[]string{bucketARNPrefix + "my-bucket"}, nil, true},
-		{[]string{bucketARNPrefix + "my-bucket/Asia/*"}, nil, true},
-		{[]string{bucketARNPrefix + "my-bucket/Asia/India/*"}, nil, true},
-	}
-	for i, testCase := range testCases {
-		err := isValidResources(set.CreateStringSet(testCase.resources...))
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-	}
-}
-
-// Tests validate principals validator.
-func TestIsValidPrincipals(t *testing.T) {
-	testCases := []struct {
-		// input.
-		principals []string
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Principals.
-		// Test case - 1.
-		// Empty Principals list.
-		{[]string{}, errors.New("Principal cannot be empty"), false},
-		// Test case - 2.
-		// "*" is the only valid principal.
-		{[]string{"my-principal"}, errors.New("Unsupported principals found: ‘set.StringSet{\"my-principal\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 3.
-		{[]string{"*", "111122233"}, errors.New("Unsupported principals found: ‘set.StringSet{\"111122233\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 4.
-		// Test case with valid principal value.
-		{[]string{"*"}, nil, true},
-	}
-	for i, testCase := range testCases {
-		u := policy.User{
-			AWS: set.CreateStringSet(testCase.principals...),
-		}
-		err := isValidPrincipals(u)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-	}
-}
-
-// getEmptyConditionMap - returns a function that generates a
-// condition key map for a given key.
-func getEmptyConditionMap(conditionKey string) func() policy.ConditionMap {
-	emptyConditonGenerator := func() policy.ConditionMap {
-		emptyMap := make(policy.ConditionKeyMap)
-		conditions := make(policy.ConditionMap)
-		conditions[conditionKey] = emptyMap
-		return conditions
-	}
-	return emptyConditonGenerator
-}
-
-// Tests validate policy.Statement condition validator.
-func TestIsValidConditions(t *testing.T) {
-	// returns empty conditions map.
-	setEmptyConditions := func() policy.ConditionMap {
-		return make(policy.ConditionMap)
-	}
-
-	// returns map with the "StringEquals" set to empty map.
-	setEmptyStringEquals := getEmptyConditionMap("StringEquals")
-
-	// returns map with the "StringNotEquals" set to empty map.
-	setEmptyStringNotEquals := getEmptyConditionMap("StringNotEquals")
-
-	// returns map with the "StringLike" set to empty map.
-	setEmptyStringLike := getEmptyConditionMap("StringLike")
-
-	// returns map with the "StringNotLike" set to empty map.
-	setEmptyStringNotLike := getEmptyConditionMap("StringNotLike")
-
-	// returns map with the "IpAddress" set to empty map.
-	setEmptyIPAddress := getEmptyConditionMap("IpAddress")
-
-	// returns map with "NotIpAddress" set to empty map.
-	setEmptyNotIPAddress := getEmptyConditionMap("NotIpAddress")
-
-	// Generate conditions.
-	generateConditions := func(key1, key2, value string) policy.ConditionMap {
-		innerMap := make(policy.ConditionKeyMap)
-		innerMap[key2] = set.CreateStringSet(value)
-		conditions := make(policy.ConditionMap)
-		conditions[key1] = innerMap
-		return conditions
-	}
-
-	// generate ambigious conditions.
-	generateAmbigiousConditions := func() policy.ConditionMap {
-		prefixMap := make(policy.ConditionKeyMap)
-		prefixMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
-		conditions["StringEquals"] = prefixMap
-		conditions["StringNotEquals"] = prefixMap
-		return conditions
-	}
-
-	// generate valid and non valid type in the condition map.
-	generateValidInvalidConditions := func() policy.ConditionMap {
-		innerMap := make(policy.ConditionKeyMap)
-		innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
-		conditions["StringEquals"] = innerMap
-		conditions["InvalidType"] = innerMap
-		return conditions
-	}
-
-	// generate valid and invalid keys for valid types in the same condition map.
-	generateValidInvalidConditionKeys := func() policy.ConditionMap {
-		innerMapValid := make(policy.ConditionKeyMap)
-		innerMapValid["s3:prefix"] = set.CreateStringSet("Asia/")
-		innerMapInValid := make(map[string]set.StringSet)
-		innerMapInValid["s3:invalid"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
-		conditions["StringEquals"] = innerMapValid
-		conditions["StringEquals"] = innerMapInValid
-		return conditions
-	}
-
-	// List of Conditions used for test cases.
-	testConditions := []policy.ConditionMap{
-		generateConditions("StringValues", "s3:max-keys", "100"),
-		generateConditions("StringEquals", "s3:Object", "100"),
-		generateAmbigiousConditions(),
-		generateValidInvalidConditions(),
-		generateValidInvalidConditionKeys(),
-		setEmptyConditions(),
-		setEmptyStringEquals(),
-		setEmptyStringNotEquals(),
-		setEmptyStringLike(),
-		setEmptyStringNotLike(),
-		setEmptyIPAddress(),
-		setEmptyNotIPAddress(),
-		generateConditions("StringEquals", "s3:prefix", "Asia/"),
-		generateConditions("StringEquals", "s3:max-keys", "100"),
-		generateConditions("StringNotEquals", "s3:prefix", "Asia/"),
-		generateConditions("StringNotEquals", "s3:max-keys", "100"),
-	}
-
-	getObjectActionSet := set.CreateStringSet("s3:GetObject")
-	roBucketActionSet := set.CreateStringSet(readOnlyBucketActions...)
-	maxKeysConditionErr := fmt.Errorf("Unsupported condition key %s for the given actions %s, "+
-		"please validate your policy document", "s3:max-keys", getObjectActionSet)
-	testCases := []struct {
-		inputActions   set.StringSet
-		inputCondition policy.ConditionMap
-		// expected result.
-		expectedErr error
-		// flag indicating whether test should pass.
-		shouldPass bool
-	}{
-		// Malformed conditions.
-		// Test case - 1.
-		// "StringValues" is an invalid type.
-		{roBucketActionSet, testConditions[0], fmt.Errorf("Unsupported condition type 'StringValues', " +
-			"please validate your policy document"), false},
-		// Test case - 2.
-		// "s3:Object" is an invalid key.
-		{roBucketActionSet, testConditions[1], fmt.Errorf("Unsupported condition key " +
-			"'StringEquals', please validate your policy document"), false},
-		// Test case - 3.
-		// Test case with Ambigious conditions set.
-		{roBucketActionSet, testConditions[2], fmt.Errorf("Ambigious condition values for key 's3:prefix', " +
-			"please validate your policy document"), false},
-		// Test case - 4.
-		// Test case with valid and invalid condition types.
-		{roBucketActionSet, testConditions[3], fmt.Errorf("Unsupported condition type 'InvalidType', " +
-			"please validate your policy document"), false},
-		// Test case - 5.
-		// Test case with valid and invalid condition keys.
-		{roBucketActionSet, testConditions[4], fmt.Errorf("Unsupported condition key 'StringEquals', " +
-			"please validate your policy document"), false},
-		// Test cases with valid conditions.
-		// Test case - 6.
-		{roBucketActionSet, testConditions[5], nil, true},
-		// Test case - 7.
-		{roBucketActionSet, testConditions[6], nil, true},
-		// Test case - 8.
-		{roBucketActionSet, testConditions[7], nil, true},
-		// Test case - 9.
-		{roBucketActionSet, testConditions[8], nil, true},
-		// Test case - 10.
-		{roBucketActionSet, testConditions[9], nil, true},
-		// Test case - 11.
-		{roBucketActionSet, testConditions[10], nil, true},
-		// Test case - 12.
-		{roBucketActionSet, testConditions[11], nil, true},
-		// Test case - 13.
-		{roBucketActionSet, testConditions[12], nil, true},
-		// Test case - 11.
-		{roBucketActionSet, testConditions[13], nil, true},
-		// Test case - 12.
-		{roBucketActionSet, testConditions[14], nil, true},
-		// Test case - 13.
-		{getObjectActionSet, testConditions[15], maxKeysConditionErr, false},
-	}
-	for i, testCase := range testCases {
-		actualErr := isValidConditions(testCase.inputActions, testCase.inputCondition)
-		if actualErr != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, actualErr.Error())
-		}
-		if actualErr == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.expectedErr.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if actualErr != nil && !testCase.shouldPass {
-			if actualErr.Error() != testCase.expectedErr.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.expectedErr.Error(), actualErr.Error())
-			}
-		}
-	}
-}
-
-// Tests validate Policy Action and Resource fields.
-func TestCheckbucketPolicyResources(t *testing.T) {
-	// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
-	setValidPrefixActions := func(statements []policy.Statement) []policy.Statement {
-		statements[0].Actions = set.CreateStringSet([]string{"s3:DeleteObject", "s3:PutObject"}...)
-		return statements
-	}
-	// contracting policy statement with recursive resources.
-	// should result in ErrMalformedPolicy
-	setRecurseResource := func(statements []policy.Statement) []policy.Statement {
-		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}...)
-		return statements
-	}
-
-	// constructing policy statement with lexically close characters.
-	// should not result in ErrMalformedPolicy
-	setResourceLexical := func(statements []policy.Statement) []policy.Statement {
-		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}...)
-		return statements
-	}
-
-	// List of bucketPolicy used for tests.
-	bucketAccessPolicies := []policy.BucketAccessPolicy{
-		// bucketPolicy - 1.
-		// Contains valid read only policy statement.
-		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
-		// bucketPolicy - 2.
-		// Contains valid read-write only policy statement.
-		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
-		// bucketPolicy - 3.
-		// Contains valid write only policy statement.
-		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
-		// bucketPolicy - 4.
-		// Contains invalidPrefixActions.
-		// Since resourcePrefix is not to the bucket-name, it return ErrMalformedPolicy.
-		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket-fail", "Asia/India/")},
-		// bucketPolicy - 5.
-		// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
-		// but bucket part of the resource is not equal to the bucket name.
-		// this results in return of ErrMalformedPolicy.
-		{Version: "1.0", Statements: setValidPrefixActions(getWriteOnlyStatement("minio-bucket-fail", "Asia/India/"))},
-		// bucketPolicy - 6.
-		// contracting policy statement with recursive resources.
-		// should result in ErrMalformedPolicy
-		{Version: "1.0", Statements: setRecurseResource(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "")))},
-		// BucketPolciy - 7.
-		// constructing policy statement with non recursive but
-		// lexically close resources.
-		// should result in ErrNone.
-		{Version: "1.0", Statements: setResourceLexical(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "oo")))},
-	}
-
-	testCases := []struct {
-		inputPolicy policy.BucketAccessPolicy
-		// expected results.
-		apiErrCode APIErrorCode
-		// Flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Test case - 1.
-		{bucketAccessPolicies[0], ErrNone, true},
-		// Test case - 2.
-		{bucketAccessPolicies[1], ErrNone, true},
-		// Test case - 3.
-		{bucketAccessPolicies[2], ErrNone, true},
-		// Test case - 4.
-		// contains invalidPrefixActions (check bucket-policy-parser.go).
-		// Resource prefix will not be equal to the bucket name in this case.
-		{bucketAccessPolicies[3], ErrMalformedPolicy, false},
-		// Test case - 5.
-		// actions contain invalidPrefixActions (check bucket-policy-parser.go).
-		// Resource prefix bucket part is not equal to the bucket name in this case.
-		{bucketAccessPolicies[4], ErrMalformedPolicy, false},
-		// Test case - 6.
-		// contracting policy statement with recursive resources.
-		// should result in ErrPolicyNesting.
-		{bucketAccessPolicies[5], ErrPolicyNesting, false},
-		// Test case - 7.
-		// constructing policy statement with lexically close
-		// characters.
-		// should result in ErrNone.
-		{bucketAccessPolicies[6], ErrNone, true},
-	}
-	for i, testCase := range testCases {
-		apiErrCode := checkBucketPolicyResources("minio-bucket", testCase.inputPolicy)
-		if apiErrCode != ErrNone && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with Errocode %v", i+1, apiErrCode)
-		}
-		if apiErrCode == ErrNone && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with ErrCode %v, but passed instead", i+1, testCase.apiErrCode)
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if apiErrCode != ErrNone && !testCase.shouldPass {
-			if testCase.apiErrCode != apiErrCode {
-				t.Errorf("Test %d: Expected to fail with error code %v, but instead failed with error code %v", i+1, testCase.apiErrCode, apiErrCode)
-			}
-		}
-	}
-}
-
-// Tests validate parsing of BucketAccessPolicy.
-func TestParseBucketPolicy(t *testing.T) {
-	// set Unsupported Actions.
-	setUnsupportedActions := func(statements []policy.Statement) []policy.Statement {
-		// "s3:DeleteEverything"" is an Unsupported Action.
-		statements[0].Actions = set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...)
-		return statements
-	}
-	// set unsupported Effect.
-	setUnsupportedEffect := func(statements []policy.Statement) []policy.Statement {
-		// Effect "Don't allow" is Unsupported.
-		statements[0].Effect = "DontAllow"
-		return statements
-	}
-	// set unsupported principals.
-	setUnsupportedPrincipals := func(statements []policy.Statement) []policy.Statement {
-		// "User1111"" is an Unsupported Principal.
-		statements[0].Principal = policy.User{
-			AWS: set.CreateStringSet([]string{"*", "User1111"}...),
-		}
-		return statements
-	}
-	// set unsupported Resources.
-	setUnsupportedResources := func(statements []policy.Statement) []policy.Statement {
-		// "s3:DeleteEverything"" is an Unsupported Action.
-		statements[0].Resources = set.CreateStringSet([]string{"my-resource"}...)
-		return statements
-	}
-	// List of bucketPolicy used for test cases.
-	bucketAccesPolicies := []policy.BucketAccessPolicy{
-		// bucketPolicy - 0.
-		// bucketPolicy statement empty.
-		{Version: "1.0"},
-		// bucketPolicy - 1.
-		// bucketPolicy version empty.
-		{Version: "", Statements: []policy.Statement{}},
-		// bucketPolicy - 2.
-		// Readonly bucketPolicy.
-		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
-		// bucketPolicy - 3.
-		// Read-Write bucket policy.
-		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
-		// bucketPolicy - 4.
-		// Write only bucket policy.
-		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
-		// bucketPolicy - 5.
-		// bucketPolicy statement contains unsupported action.
-		{Version: "1.0", Statements: setUnsupportedActions(getReadOnlyStatement("minio-bucket", ""))},
-		// bucketPolicy - 6.
-		// bucketPolicy statement contains unsupported Effect.
-		{Version: "1.0", Statements: setUnsupportedEffect(getReadWriteStatement("minio-bucket", "Asia/"))},
-		// bucketPolicy - 7.
-		// bucketPolicy statement contains unsupported Principal.
-		{Version: "1.0", Statements: setUnsupportedPrincipals(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
-		// bucketPolicy - 8.
-		// bucketPolicy statement contains unsupported Resource.
-		{Version: "1.0", Statements: setUnsupportedResources(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
-	}
-
-	testCases := []struct {
-		inputPolicy policy.BucketAccessPolicy
-		// expected results.
-		expectedPolicy policy.BucketAccessPolicy
-		err            error
-		// Flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Test case - 1.
-		// bucketPolicy statement empty.
-		{bucketAccesPolicies[0], policy.BucketAccessPolicy{}, errors.New("Policy statement cannot be empty"), false},
-		// Test case - 2.
-		// bucketPolicy version empty.
-		{bucketAccesPolicies[1], policy.BucketAccessPolicy{}, errors.New("Policy version cannot be empty"), false},
-		// Test case - 3.
-		// Readonly bucketPolicy.
-		{bucketAccesPolicies[2], bucketAccesPolicies[2], nil, true},
-		// Test case - 4.
-		// Read-Write bucket policy.
-		{bucketAccesPolicies[3], bucketAccesPolicies[3], nil, true},
-		// Test case - 5.
-		// Write only bucket policy.
-		{bucketAccesPolicies[4], bucketAccesPolicies[4], nil, true},
-		// Test case - 6.
-		// bucketPolicy statement contains unsupported action.
-		{bucketAccesPolicies[5], bucketAccesPolicies[5], fmt.Errorf("Unsupported actions found: ‘set.StringSet{\"s3:DeleteEverything\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 7.
-		// bucketPolicy statement contains unsupported Effect.
-		{bucketAccesPolicies[6], bucketAccesPolicies[6], fmt.Errorf("Unsupported Effect found: ‘DontAllow’, please validate your policy document"), false},
-		// Test case - 8.
-		// bucketPolicy statement contains unsupported Principal.
-		{bucketAccesPolicies[7], bucketAccesPolicies[7], fmt.Errorf("Unsupported principals found: ‘set.StringSet{\"User1111\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 9.
-		// bucketPolicy statement contains unsupported Resource.
-		{bucketAccesPolicies[8], bucketAccesPolicies[8], fmt.Errorf("Unsupported resource style found: ‘my-resource’, please validate your policy document"), false},
-	}
-	for i, testCase := range testCases {
-		var buffer bytes.Buffer
-		encoder := json.NewEncoder(&buffer)
-		err := encoder.Encode(testCase.inputPolicy)
-		if err != nil {
-			t.Fatalf("Test %d: Couldn't Marshal bucket policy %s", i+1, err)
-		}
-
-		var actualAccessPolicy = policy.BucketAccessPolicy{}
-		err = parseBucketPolicy(&buffer, &actualAccessPolicy)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-		// Test passes as expected, but the output values are verified for correctness here.
-		if err == nil && testCase.shouldPass {
-			if !reflect.DeepEqual(testCase.expectedPolicy, actualAccessPolicy) {
-				t.Errorf("Test %d: The expected statements from resource statement generator doesn't match the actual statements", i+1)
-			}
-		}
-	}
-}
-
-func TestAWSRefererCondition(t *testing.T) {
-	resource := set.CreateStringSet([]string{
-		fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/"+"Asia"+"*"),
-	}...)
-
-	conditionsKeyMap := make(policy.ConditionKeyMap)
-	conditionsKeyMap.Add("aws:Referer",
-		set.CreateStringSet("www.example.com",
-			"http://www.example.com"))
-
-	requestConditionMap := make(policy.ConditionKeyMap)
-	requestConditionMap["referer"] = set.CreateStringSet("www.example.com")
-
-	testCases := []struct {
-		effect       string
-		conditionKey string
-		match        bool
-	}{
-		{
-			effect:       "Allow",
-			conditionKey: "StringLike",
-			match:        true,
-		},
-		{
-			effect:       "Allow",
-			conditionKey: "StringNotLike",
-			match:        false,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "StringLike",
-			match:        true,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "StringNotLike",
-			match:        false,
-		},
-	}
-
-	for i, test := range testCases {
-		conditions := make(policy.ConditionMap)
-		conditions[test.conditionKey] = conditionsKeyMap
-
-		allowStatement := policy.Statement{
-			Sid:    "Testing AWS referer condition",
-			Effect: test.effect,
-			Principal: policy.User{
-				AWS: set.CreateStringSet("*"),
-			},
-			Resources:  resource,
-			Conditions: conditions,
-		}
-
-		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
-			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
-				i+1, test.match, result)
-		}
-	}
-}
-
-func TestAWSSourceIPCondition(t *testing.T) {
-	resource := set.CreateStringSet([]string{
-		fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/"+"Asia"+"*"),
-	}...)
-
-	conditionsKeyMap := make(policy.ConditionKeyMap)
-	// Test both IPv4 and IPv6 addresses.
-	conditionsKeyMap.Add("aws:SourceIp",
-		set.CreateStringSet("54.240.143.0/24",
-			"2001:DB8:1234:5678::/64"))
-
-	requestConditionMap := make(policy.ConditionKeyMap)
-	requestConditionMap["ip"] = set.CreateStringSet("54.240.143.2")
-
-	testCases := []struct {
-		effect       string
-		conditionKey string
-		match        bool
-	}{
-		{
-			effect:       "Allow",
-			conditionKey: "IpAddress",
-			match:        true,
-		},
-		{
-			effect:       "Allow",
-			conditionKey: "NotIpAddress",
-			match:        false,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "IpAddress",
-			match:        true,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "NotIpAddress",
-			match:        false,
-		},
-	}
-
-	for i, test := range testCases {
-		conditions := make(policy.ConditionMap)
-		conditions[test.conditionKey] = conditionsKeyMap
-
-		allowStatement := policy.Statement{
-			Sid:    "Testing AWS referer condition",
-			Effect: test.effect,
-			Principal: policy.User{
-				AWS: set.CreateStringSet("*"),
-			},
-			Resources:  resource,
-			Conditions: conditions,
-		}
-
-		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
-			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
-				i+1, test.match, result)
-		}
-	}
-}

cmd/bucket-policy.go

@@ -1,207 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"reflect"
-	"sync"
-
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio/pkg/errors"
-	"github.com/minio/minio/pkg/hash"
-)
-
-const (
-	// Static prefix to be used while constructing bucket ARN.
-	// refer to S3 docs for more info.
-	bucketARNPrefix = "arn:aws:s3:::"
-
-	// Bucket policy config name.
-	bucketPolicyConfig = "policy.json"
-)
-
-// Global bucket policies list, policies are enforced on each bucket looking
-// through the policies here.
-type bucketPolicies struct {
-	rwMutex *sync.RWMutex
-
-	// Collection of 'bucket' policies.
-	bucketPolicyConfigs map[string]policy.BucketAccessPolicy
-}
-
-// Fetch bucket policy for a given bucket.
-func (bp bucketPolicies) GetBucketPolicy(bucket string) policy.BucketAccessPolicy {
-	bp.rwMutex.RLock()
-	defer bp.rwMutex.RUnlock()
-	return bp.bucketPolicyConfigs[bucket]
-}
-
-// Set a new bucket policy for a bucket, this operation will overwrite
-// any previous bucket policies for the bucket.
-func (bp *bucketPolicies) SetBucketPolicy(bucket string, newpolicy policy.BucketAccessPolicy) error {
-	bp.rwMutex.Lock()
-	defer bp.rwMutex.Unlock()
-
-	if reflect.DeepEqual(newpolicy, emptyBucketPolicy) {
-		return errInvalidArgument
-	}
-	bp.bucketPolicyConfigs[bucket] = newpolicy
-
-	return nil
-}
-
-// Delete bucket policy from struct for a given bucket.
-func (bp *bucketPolicies) DeleteBucketPolicy(bucket string) error {
-	bp.rwMutex.Lock()
-	defer bp.rwMutex.Unlock()
-	delete(bp.bucketPolicyConfigs, bucket)
-	return nil
-}
-
-// Intialize all bucket policies.
-func initBucketPolicies(objAPI ObjectLayer) (*bucketPolicies, error) {
-	if objAPI == nil {
-		return nil, errInvalidArgument
-	}
-
-	// List buckets to proceed loading all notification configuration.
-	buckets, err := objAPI.ListBuckets(context.Background())
-	if err != nil {
-		return nil, errors.Cause(err)
-	}
-
-	policies := make(map[string]policy.BucketAccessPolicy)
-	// Loads bucket policy.
-	for _, bucket := range buckets {
-		bp, pErr := ReadBucketPolicy(bucket.Name, objAPI)
-		if pErr != nil {
-			// net.Dial fails for rpc client or any
-			// other unexpected errors during net.Dial.
-			if !errors.IsErrIgnored(pErr, errDiskNotFound) {
-				if !isErrBucketPolicyNotFound(pErr) {
-					return nil, errors.Cause(pErr)
-				}
-			}
-			// Continue to load other bucket policies if possible.
-			continue
-		}
-		policies[bucket.Name] = bp
-	}
-
-	// Return all bucket policies.
-	return &bucketPolicies{
-		rwMutex:             &sync.RWMutex{},
-		bucketPolicyConfigs: policies,
-	}, nil
-}
-
-// readBucketPolicyJSON - reads bucket policy for an input bucket, returns BucketPolicyNotFound
-// if bucket policy is not found.
-func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader io.Reader, err error) {
-	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-
-	var buffer bytes.Buffer
-	err = objAPI.GetObject(context.Background(), minioMetaBucket, policyPath, 0, -1, &buffer, "")
-	if err != nil {
-		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, PolicyNotFound{Bucket: bucket}
-		}
-		errorIf(err, "Unable to load policy for the bucket %s.", bucket)
-		return nil, errors.Cause(err)
-	}
-
-	return &buffer, nil
-}
-
-// ReadBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
-// if bucket policy is not found. This function also parses the bucket policy into an object.
-func ReadBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPolicy, error) {
-	// Read bucket policy JSON.
-	bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI)
-	if err != nil {
-		return emptyBucketPolicy, err
-	}
-
-	// Parse the saved policy.
-	var bp policy.BucketAccessPolicy
-	if err = parseBucketPolicy(bucketPolicyReader, &bp); err != nil {
-		return emptyBucketPolicy, err
-
-	}
-	return bp, nil
-}
-
-// removeBucketPolicy - removes any previously written bucket policy. Returns BucketPolicyNotFound
-// if no policies are found.
-func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
-	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-	err := objAPI.DeleteObject(context.Background(), minioMetaBucket, policyPath)
-	if err != nil {
-		err = errors.Cause(err)
-		if _, ok := err.(ObjectNotFound); ok {
-			return BucketPolicyNotFound{Bucket: bucket}
-		}
-		return err
-	}
-	return nil
-}
-
-// writeBucketPolicy - save a bucket policy that is assumed to be validated.
-func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy policy.BucketAccessPolicy) error {
-	buf, err := json.Marshal(bpy)
-	if err != nil {
-		errorIf(err, "Unable to marshal bucket policy '%#v' to JSON", bpy)
-		return err
-	}
-	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
-	if err != nil {
-		errorIf(err, "Unable to set policy for the bucket %s", bucket)
-		return errors.Cause(err)
-	}
-
-	if _, err = objAPI.PutObject(context.Background(), minioMetaBucket, policyPath, hashReader, nil); err != nil {
-		errorIf(err, "Unable to set policy for the bucket %s", bucket)
-		return errors.Cause(err)
-	}
-	return nil
-}
-
-// persistAndNotifyBucketPolicyChange - takes a policyChange argument,
-// persists it to storage, and notify nodes in the cluster about the
-// change. In-memory state is updated in response to the notification.
-func persistAndNotifyBucketPolicyChange(bucket string, isRemove bool, bktPolicy policy.BucketAccessPolicy, objAPI ObjectLayer) error {
-	if isRemove {
-		err := removeBucketPolicy(bucket, objAPI)
-		if err != nil {
-			return err
-		}
-	} else {
-		if reflect.DeepEqual(bktPolicy, emptyBucketPolicy) {
-			return errInvalidArgument
-		}
-		if err := writeBucketPolicy(bucket, objAPI, bktPolicy); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

cmd/build-constants.go

@@ -16,13 +16,14 @@
 
 package cmd
 
-import "go/build"
-
 // DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
 // set through ‘buildscripts/gen-ldflags.go’.
 var (
 	// GOPATH - GOPATH value at the time of build.
-	GOPATH = build.Default.GOPATH
+	GOPATH = ""
+
+	// GOROOT - GOROOT value at the time of build.
+	GOROOT = ""
 
 	// Go get development tag.
 	goGetTag = "DEVELOPMENT.GOGET"

cmd/certs.go

@@ -17,12 +17,16 @@
 package cmd
 
 import (
+	"bytes"
+	"crypto"
+	"crypto/ecdsa"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
-	"fmt"
 	"io/ioutil"
 	"os"
+
+	"github.com/minio/minio/pkg/certs"
 )
 
 // TLSPrivateKeyPassword is the environment variable which contains the password used
@@ -37,24 +41,27 @@ func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 		return nil, err
 	}
 
+	// Trimming leading and tailing white spaces.
+	data = bytes.TrimSpace(data)
+
 	// Parse all certs in the chain.
 	current := data
 	for len(current) > 0 {
 		var pemBlock *pem.Block
 		if pemBlock, current = pem.Decode(current); pemBlock == nil {
-			return nil, fmt.Errorf("Could not read PEM block from file %s", certFile)
+			return nil, uiErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
 		}
 
 		var x509Cert *x509.Certificate
 		if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
-			return nil, err
+			return nil, uiErrSSLUnexpectedData(err)
 		}
 
 		x509Certs = append(x509Certs, x509Cert)
 	}
 
 	if len(x509Certs) == 0 {
-		return nil, fmt.Errorf("Empty public certificate file %s", certFile)
+		return nil, uiErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
 	}
 
 	return x509Certs, nil
@@ -79,10 +86,11 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 		return nil, nil
 	}
 
-	rootCAs, err := x509.SystemCertPool()
-	if err != nil {
-		// In some systems like Windows, system cert pool is not supported.
-		// Hence we create a new cert pool.
+	rootCAs, _ := x509.SystemCertPool()
+	if rootCAs == nil {
+		// In some systems (like Windows) system cert pool is
+		// not supported or no certificates are present on the
+		// system - so we create a new cert pool.
 		rootCAs = x509.NewCertPool()
 	}
 
@@ -105,31 +113,44 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	certPEMBlock, err := ioutil.ReadFile(certFile)
 	if err != nil {
-		return tls.Certificate{}, fmt.Errorf("TLS: failed to read cert file: %v", err)
+		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
 	}
 	keyPEMBlock, err := ioutil.ReadFile(keyFile)
 	if err != nil {
-		return tls.Certificate{}, fmt.Errorf("TLS: failed to read private key: %v", err)
+		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
 	}
 	key, rest := pem.Decode(keyPEMBlock)
 	if len(rest) > 0 {
-		return tls.Certificate{}, fmt.Errorf("TLS: private key contains additional data")
+		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
 	}
 	if x509.IsEncryptedPEMBlock(key) {
 		password, ok := os.LookupEnv(TLSPrivateKeyPassword)
 		if !ok {
-			return tls.Certificate{}, fmt.Errorf("TLS: private key is encrypted but no password is present - set env var: %s", TLSPrivateKeyPassword)
+			return tls.Certificate{}, uiErrSSLNoPassword(nil)
 		}
 		decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
 		if decErr != nil {
-			return tls.Certificate{}, fmt.Errorf("TLS: failed to decrypt private key: %v", decErr)
+			return tls.Certificate{}, uiErrSSLWrongPassword(decErr)
 		}
 		keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
 	}
-	return tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+	cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+	if err != nil {
+		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg(err.Error())
+	}
+	// Ensure that the private key is not a P-384 or P-521 EC key.
+	// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
+	if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
+		if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
+			if name := pub.Params().Name; name == "P-384" || name == "P-521" { // unfortunately there is no cleaner way to check
+				return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
+			}
+		}
+	}
+	return cert, nil
 }
 
-func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
+func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, c *certs.Certs, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
 		return nil, nil, nil, false, nil
 	}
@@ -138,17 +159,15 @@ func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsC
 		return nil, nil, nil, false, err
 	}
 
-	var cert tls.Certificate
-	if cert, err = loadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
+	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
+	if err != nil {
 		return nil, nil, nil, false, err
 	}
 
-	tlsCert = &cert
-
 	if rootCAs, err = getRootCAs(getCADir()); err != nil {
 		return nil, nil, nil, false, err
 	}
 
 	secureConn = true
-	return x509Certs, rootCAs, tlsCert, secureConn, nil
+	return x509Certs, rootCAs, c, secureConn, nil
 }

cmd/common-main.go

@@ -17,15 +17,23 @@
 package cmd
 
 import (
+	"context"
 	"errors"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
+	etcd "github.com/coreos/etcd/clientv3"
+
 	"github.com/minio/cli"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
 // Check for updates and print a notification message
@@ -33,21 +41,43 @@ func checkUpdate(mode string) {
 	// Its OK to ignore any errors during doUpdate() here.
 	if updateMsg, _, currentReleaseTime, latestReleaseTime, err := getUpdateInfo(2*time.Second, mode); err == nil {
 		if globalInplaceUpdateDisabled {
-			log.Println(updateMsg)
+			logger.StartupMessage(updateMsg)
 		} else {
-			log.Println(prepareUpdateMessage("Run `minio update`", latestReleaseTime.Sub(currentReleaseTime)))
+			logger.StartupMessage(prepareUpdateMessage("Run `minio update`", latestReleaseTime.Sub(currentReleaseTime)))
 		}
 	}
 }
 
+// Initialize and load config from remote etcd or local config directory
 func initConfig() {
-	// Config file does not exist, we create it fresh and return upon success.
+	if globalEtcdClient != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
+		cancel()
+		// This means there are no entries in etcd with config file
+		// So create a new config
+		if err == nil && resp.Count == 0 {
+			logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
+			logger.Info("Created minio configuration file successfully at %v", globalEtcdClient.Endpoints())
+		} else {
+			// This means there is an entry in etcd, update it if required and proceed
+			if err == nil && resp.Count > 0 {
+				logger.FatalIf(migrateConfig(), "Config migration failed.")
+				logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
+			} else {
+				logger.FatalIf(err, "Unable to load config version: '%s'.", serverConfigVersion)
+			}
+		}
+		return
+	}
+
 	if isFile(getConfigFile()) {
-		fatalIf(migrateConfig(), "Config migration failed.")
-		fatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
+		logger.FatalIf(migrateConfig(), "Config migration failed")
+		logger.FatalIf(loadConfig(), "Unable to load the configuration file")
 	} else {
-		fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
-		log.Println("Created minio configuration file successfully at " + getConfigDir())
+		// Config file does not exist, we create it fresh and return upon success.
+		logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time")
+		logger.Info("Created minio configuration file successfully at " + getConfigDir())
 	}
 }
 
@@ -70,17 +100,17 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 		// default config directory.
 		configDir = getConfigDir()
 		if configDir == "" {
-			fatalIf(errors.New("missing option"), "config-dir option must be provided.")
+			logger.FatalIf(errors.New("missing option"), "config-dir option must be provided")
 		}
 	}
 
 	if configDir == "" {
-		fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
+		logger.FatalIf(errors.New("empty directory"), "Configuration directory cannot be empty")
 	}
 
 	// Disallow relative paths, figure out absolute paths.
 	configDirAbs, err := filepath.Abs(configDir)
-	fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
+	logger.FatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
 	setConfigDir(configDirAbs)
 }
 
@@ -94,7 +124,9 @@ func handleCommonEnvVars() {
 	secretKey := os.Getenv("MINIO_SECRET_KEY")
 	if accessKey != "" && secretKey != "" {
 		cred, err := auth.CreateCredentials(accessKey, secretKey)
-		fatalIf(err, "Invalid access/secret Key set in environment.")
+		if err != nil {
+			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
+		}
 
 		// credential Envs are set globally.
 		globalIsEnvCreds = true
@@ -102,9 +134,9 @@ func handleCommonEnvVars() {
 	}
 
 	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
-		browserFlag, err := ParseBrowserFlag(browser)
+		browserFlag, err := ParseBoolFlag(browser)
 		if err != nil {
-			fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
+			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Unable to validate MINIO_BROWSER environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -117,31 +149,75 @@ func handleCommonEnvVars() {
 	if traceFile != "" {
 		var err error
 		globalHTTPTraceFile, err = os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
-		fatalIf(err, "error opening file %s", traceFile)
+		logger.FatalIf(err, "error opening file %s", traceFile)
 	}
 
-	globalDomainName = os.Getenv("MINIO_DOMAIN")
-	if globalDomainName != "" {
-		globalIsEnvDomainName = true
+	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	if ok {
+		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
+		var err error
+		globalEtcdClient, err = etcd.New(etcd.Config{
+			Endpoints:         etcdEndpoints,
+			DialTimeout:       defaultDialTimeout,
+			DialKeepAliveTime: defaultDialKeepAlive,
+		})
+		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+	}
+
+	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
+
+	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if ok {
+		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
+		globalDomainIPs = set.NewStringSet()
+		for i, ip := range minioEndpoints {
+			if net.ParseIP(ip) == nil {
+				logger.FatalIf(errInvalidArgument, "Unable to initialize Minio server with invalid MINIO_PUBLIC_IPS[%d]: %s", i, ip)
+			}
+			globalDomainIPs.Add(ip)
+		}
+	}
+	if globalDomainName != "" && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+		var err error
+		globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIPs, globalMinioPort, globalEtcdClient)
+		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainName)
 	}
 
 	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
 		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
-		fatalIf(err, "Invalid value set in environment variable MINIO_CACHE_DRIVES %s.", drives)
+		if err != nil {
+			logger.Fatal(err, "Unable to parse MINIO_CACHE_DRIVES value (%s)", drives)
+		}
 		globalCacheDrives = driveList
 		globalIsDiskCacheEnabled = true
 	}
+
 	if excludes := os.Getenv("MINIO_CACHE_EXCLUDE"); excludes != "" {
 		excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter))
-		fatalIf(err, "Invalid value set in environment variable MINIO_CACHE_EXCLUDE %s.", excludes)
+		if err != nil {
+			logger.Fatal(err, "Unable to parse MINIO_CACHE_EXCLUDE value (`%s`)", excludes)
+		}
 		globalCacheExcludes = excludeList
 	}
+
 	if expiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); expiryStr != "" {
 		expiry, err := strconv.Atoi(expiryStr)
-		fatalIf(err, "Invalid value set in environment variable MINIO_CACHE_EXPIRY %s.", expiryStr)
+		if err != nil {
+			logger.Fatal(uiErrInvalidCacheExpiryValue(err), "Unable to parse MINIO_CACHE_EXPIRY value (`%s`)", expiryStr)
+		}
 		globalCacheExpiry = expiry
 	}
 
+	if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" {
+		maxUse, err := strconv.Atoi(maxUseStr)
+		if err != nil {
+			logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr)
+		}
+		// maxUse should be a valid percentage.
+		if maxUse > 0 && maxUse <= 100 {
+			globalCacheMaxUse = maxUse
+		}
+	}
 	// In place update is true by default if the MINIO_UPDATE is not set
 	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
 	// in-place update is off.
@@ -154,29 +230,39 @@ func handleCommonEnvVars() {
 		// Check for environment variables and parse into storageClass struct
 		if ssc := os.Getenv(standardStorageClassEnv); ssc != "" {
 			globalStandardStorageClass, err = parseStorageClass(ssc)
-			fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
 		}
 
 		if rrsc := os.Getenv(reducedRedundancyStorageClassEnv); rrsc != "" {
 			globalRRStorageClass, err = parseStorageClass(rrsc)
-			fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
 		}
 
 		// Validation is done after parsing both the storage classes. This is needed because we need one
 		// storage class value to deduce the correct value of the other storage class.
 		if globalRRStorageClass.Scheme != "" {
 			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
 			globalIsStorageClass = true
 		}
 
 		if globalStandardStorageClass.Scheme != "" {
 			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
 			globalIsStorageClass = true
 		}
 	}
 
 	// Get WORM environment variable.
-	globalWORMEnabled = strings.EqualFold(os.Getenv("MINIO_WORM"), "on")
+	if worm := os.Getenv("MINIO_WORM"); worm != "" {
+		wormFlag, err := ParseBoolFlag(worm)
+		if err != nil {
+			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Unable to validate MINIO_WORM environment variable")
+		}
+
+		// worm Envs are set globally, this does not represent
+		// if worm is turned off or on.
+		globalIsEnvWORM = true
+		globalWORMEnabled = bool(wormFlag)
+	}
 }

cmd/config-current.go

@@ -17,17 +17,18 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"reflect"
 	"sync"
 
+	"github.com/minio/minio/cmd/logger"
+
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
 	"github.com/minio/minio/pkg/quick"
-	"github.com/tidwall/gjson"
 )
 
 // Steps to move from version N to version N+1
@@ -39,9 +40,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "23"
+const serverConfigVersion = "26"
 
-type serverConfig = serverConfigV23
+type serverConfig = serverConfigV26
 
 var (
 	// globalServerConfig server config.
@@ -85,7 +86,13 @@ func (s *serverConfig) GetCredential() auth.Credentials {
 // SetBrowser set if browser is enabled.
 func (s *serverConfig) SetBrowser(b bool) {
 	// Set the new value.
-	s.Browser = BrowserFlag(b)
+	s.Browser = BoolFlag(b)
+}
+
+// SetWorm set if worm is enabled.
+func (s *serverConfig) SetWorm(b bool) {
+	// Set the new value.
+	s.Worm = BoolFlag(b)
 }
 
 func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
@@ -99,16 +106,22 @@ func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
 	return s.StorageClass.Standard, s.StorageClass.RRS
 }
 
-// GetCredentials get current credentials.
+// GetBrowser get current credentials.
 func (s *serverConfig) GetBrowser() bool {
 	return bool(s.Browser)
 }
 
+// GetWorm get current credentials.
+func (s *serverConfig) GetWorm() bool {
+	return bool(s.Worm)
+}
+
 // SetCacheConfig sets the current cache config
-func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int) {
+func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxuse int) {
 	s.Cache.Drives = drives
 	s.Cache.Exclude = exclude
 	s.Cache.Expiry = expiry
+	s.Cache.MaxUse = maxuse
 }
 
 // GetCacheConfig gets the current cache config
@@ -116,10 +129,19 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
 	return s.Cache
 }
 
-// Save config.
-func (s *serverConfig) Save() error {
-	// Save config file.
-	return quick.Save(getConfigFile(), s)
+// Save config file to corresponding backend
+func Save(configFile string, data interface{}) error {
+	return quick.SaveConfig(data, configFile, globalEtcdClient)
+}
+
+// Load config from backend
+func Load(configFile string, data interface{}) (quick.Config, error) {
+	return quick.LoadConfig(configFile, globalEtcdClient, data)
+}
+
+// GetVersion gets config version from backend
+func GetVersion(configFile string) (string, error) {
+	return quick.GetVersion(configFile, globalEtcdClient)
 }
 
 // Returns the string describing a difference with the given
@@ -139,6 +161,8 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "Domain configuration differs"
 	case s.StorageClass != t.StorageClass:
 		return "StorageClass configuration differs"
+	case !reflect.DeepEqual(s.Cache, t.Cache):
+		return "Cache configuration differs"
 	case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
 		return "AMQP Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
@@ -167,9 +191,12 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 }
 
 func newServerConfig() *serverConfig {
+	cred, err := auth.GetNewCredentials()
+	logger.FatalIf(err, "")
+
 	srvCfg := &serverConfig{
 		Version:    serverConfigVersion,
-		Credential: auth.MustGetNewCredentials(),
+		Credential: cred,
 		Region:     globalMinioDefaultRegion,
 		Browser:    true,
 		StorageClass: storageClassConfig{
@@ -180,6 +207,7 @@ func newServerConfig() *serverConfig {
 			Drives:  []string{},
 			Exclude: []string{},
 			Expiry:  globalCacheExpiry,
+			MaxUse:  globalCacheMaxUse,
 		},
 		Notify: notifier{},
 	}
@@ -207,6 +235,7 @@ func newServerConfig() *serverConfig {
 	srvCfg.Cache.Drives = make([]string, 0)
 	srvCfg.Cache.Exclude = make([]string, 0)
 	srvCfg.Cache.Expiry = globalCacheExpiry
+	srvCfg.Cache.MaxUse = globalCacheMaxUse
 	return srvCfg
 }
 
@@ -214,135 +243,7 @@ func newServerConfig() *serverConfig {
 // found, otherwise use default parameters
 func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfig()
-
-	// If env is set override the credentials from config file.
-	if globalIsEnvCreds {
-		srvCfg.SetCredential(globalActiveCred)
-	}
-
-	if globalIsEnvBrowser {
-		srvCfg.SetBrowser(globalIsBrowserEnabled)
-	}
-
-	if globalIsEnvRegion {
-		srvCfg.SetRegion(globalServerRegion)
-	}
-
-	if globalIsEnvDomainName {
-		srvCfg.Domain = globalDomainName
-	}
-
-	if globalIsStorageClass {
-		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
-	}
-
-	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
-	}
-
-	// hold the mutex lock before a new config is assigned.
-	// Save the new config globally.
-	// unlock the mutex.
-	globalServerConfigMu.Lock()
-	globalServerConfig = srvCfg
-	globalServerConfigMu.Unlock()
-
-	// Save config into file.
-	return globalServerConfig.Save()
-}
-
-// doCheckDupJSONKeys recursively detects duplicate json keys
-func doCheckDupJSONKeys(key, value gjson.Result) error {
-	// Key occurrences map of the current scope to count
-	// if there is any duplicated json key.
-	keysOcc := make(map[string]int)
-
-	// Holds the found error
-	var checkErr error
-
-	// Iterate over keys in the current json scope
-	value.ForEach(func(k, v gjson.Result) bool {
-		// If current key is not null, check if its
-		// value contains some duplicated keys.
-		if k.Type != gjson.Null {
-			keysOcc[k.String()]++
-			checkErr = doCheckDupJSONKeys(k, v)
-		}
-		return checkErr == nil
-	})
-
-	// Check found err
-	if checkErr != nil {
-		return errors.New(key.String() + " => " + checkErr.Error())
-	}
-
-	// Check for duplicated keys
-	for k, v := range keysOcc {
-		if v > 1 {
-			return errors.New(key.String() + " => `" + k + "` entry is duplicated")
-		}
-	}
-
-	return nil
-}
-
-// Check recursively if a key is duplicated in the same json scope
-// e.g.:
-//  `{ "key" : { "key" ..` is accepted
-//  `{ "key" : { "subkey" : "val1", "subkey": "val2" ..` throws subkey duplicated error
-func checkDupJSONKeys(json string) error {
-	// Parse config with gjson library
-	config := gjson.Parse(json)
-
-	// Create a fake rootKey since root json doesn't seem to have representation
-	// in gjson library.
-	rootKey := gjson.Result{Type: gjson.String, Str: minioConfigFile}
-
-	// Check if loaded json contains any duplicated keys
-	return doCheckDupJSONKeys(rootKey, config)
-}
-
-// getValidConfig - returns valid server configuration
-func getValidConfig() (*serverConfig, error) {
-	srvCfg := &serverConfig{
-		Region:  globalMinioDefaultRegion,
-		Browser: true,
-	}
-
-	configFile := getConfigFile()
-	if _, err := quick.Load(configFile, srvCfg); err != nil {
-		return nil, err
-	}
-
-	if srvCfg.Version != serverConfigVersion {
-		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
-	}
-
-	// Load config file json and check for duplication json keys
-	jsonBytes, err := ioutil.ReadFile(configFile)
-	if err != nil {
-		return nil, err
-	}
-	if err = checkDupJSONKeys(string(jsonBytes)); err != nil {
-		return nil, err
-	}
-
-	// Validate credential fields only when
-	// they are not set via the environment
-
-	// Error out if global is env credential is not set and config has invalid credential
-	if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
-		return nil, errors.New("invalid credential in config file " + configFile)
-	}
-
-	return srvCfg, nil
-}
-
-// loadConfig - loads a new config from disk, overrides params from env
-// if found and valid
-func loadConfig() error {
-	srvCfg, err := getValidConfig()
+	srvCfg, err := newQuickConfig(newServerConfig())
 	if err != nil {
 		return err
 	}
@@ -356,6 +257,10 @@ func loadConfig() error {
 		srvCfg.SetBrowser(globalIsBrowserEnabled)
 	}
 
+	if globalIsEnvWORM {
+		srvCfg.SetWorm(globalWORMEnabled)
+	}
+
 	if globalIsEnvRegion {
 		srvCfg.SetRegion(globalServerRegion)
 	}
@@ -369,7 +274,95 @@ func loadConfig() error {
 	}
 
 	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
+		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
+	}
+
+	// hold the mutex lock before a new config is assigned.
+	// Save the new config globally.
+	// unlock the mutex.
+	globalServerConfigMu.Lock()
+	globalServerConfig = srvCfg
+	globalServerConfigMu.Unlock()
+
+	// Save config into file.
+	return Save(getConfigFile(), globalServerConfig)
+}
+
+// newQuickConfig - initialize a new server config, with an allocated
+// quick.Config interface.
+func newQuickConfig(srvCfg *serverConfig) (*serverConfig, error) {
+	qcfg, err := quick.NewConfig(srvCfg, globalEtcdClient)
+	if err != nil {
+		return nil, err
+	}
+
+	srvCfg.Config = qcfg
+	return srvCfg, nil
+}
+
+// getValidConfig - returns valid server configuration
+func getValidConfig() (*serverConfig, error) {
+	srvCfg := &serverConfig{
+		Region:  globalMinioDefaultRegion,
+		Browser: true,
+	}
+
+	var err error
+	srvCfg, err = newQuickConfig(srvCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	configFile := getConfigFile()
+	if err = srvCfg.Load(configFile); err != nil {
+		return nil, err
+	}
+
+	if srvCfg.Version != serverConfigVersion {
+		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
+	}
+
+	// Validate credential fields only when
+	// they are not set via the environment
+	// Error out if global is env credential is not set and config has invalid credential
+	if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
+		return nil, errors.New("invalid credential in config file " + getConfigFile())
+	}
+
+	return srvCfg, nil
+}
+
+// loadConfig - loads a new config from disk, overrides params from env
+// if found and valid
+func loadConfig() error {
+	srvCfg, err := getValidConfig()
+	if err != nil {
+		return uiErrInvalidConfig(nil).Msg(err.Error())
+	}
+
+	// If env is set override the credentials from config file.
+	if globalIsEnvCreds {
+		srvCfg.SetCredential(globalActiveCred)
+	}
+
+	if globalIsEnvBrowser {
+		srvCfg.SetBrowser(globalIsBrowserEnabled)
+	}
+
+	if globalIsEnvRegion {
+		srvCfg.SetRegion(globalServerRegion)
+	}
+
+	if globalIsEnvDomainName {
+		srvCfg.Domain = globalDomainName
+	}
+
+	if globalIsStorageClass {
+		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
+	}
+
+	if globalIsDiskCacheEnabled {
+		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
 	}
 
 	// hold the mutex lock before a new config is assigned.
@@ -381,6 +374,9 @@ func loadConfig() error {
 	if !globalIsEnvBrowser {
 		globalIsBrowserEnabled = globalServerConfig.GetBrowser()
 	}
+	if !globalIsEnvWORM {
+		globalWORMEnabled = globalServerConfig.GetWorm()
+	}
 	if !globalIsEnvRegion {
 		globalServerRegion = globalServerConfig.GetRegion()
 	}
@@ -395,6 +391,7 @@ func loadConfig() error {
 		globalCacheDrives = cacheConf.Drives
 		globalCacheExcludes = cacheConf.Exclude
 		globalCacheExpiry = cacheConf.Expiry
+		globalCacheMaxUse = cacheConf.MaxUse
 	}
 	globalServerConfigMu.Unlock()
 
@@ -406,17 +403,19 @@ func loadConfig() error {
 // * Add a new target in pkg/event/target package.
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
-func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
+func getNotificationTargets(config *serverConfig) *event.TargetList {
 	targetList := event.NewTargetList()
 
 	for id, args := range config.Notify.AMQP {
 		if args.Enable {
 			newTarget, err := target.NewAMQPTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -425,10 +424,14 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewElasticsearchTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+
 			}
 		}
 	}
@@ -437,10 +440,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewKafkaTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -449,10 +454,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewMQTTTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -461,10 +468,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewMySQLTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -473,10 +482,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewNATSTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -485,10 +496,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewPostgreSQLTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -497,10 +510,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewRedisTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -509,10 +524,11 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget := target.NewWebhookTarget(id, args)
 			if err := targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
 
-	return targetList, nil
+	return targetList
 }

cmd/config-current_test.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
-	"github.com/tidwall/gjson"
 )
 
 func TestServerConfig(t *testing.T) {
@@ -51,7 +50,7 @@ func TestServerConfig(t *testing.T) {
 	}
 
 	// Attempt to save.
-	if err := globalServerConfig.Save(); err != nil {
+	if err := globalServerConfig.Save(getConfigFile()); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 
@@ -126,34 +125,6 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	}
 }
 
-func TestCheckDupJSONKeys(t *testing.T) {
-	testCases := []struct {
-		json       string
-		shouldPass bool
-	}{
-		{`{}`, true},
-		{`{"version" : "13"}`, true},
-		{`{"version" : "13", "version": "14"}`, false},
-		{`{"version" : "13", "credential": {"accessKey": "12345"}}`, true},
-		{`{"version" : "13", "credential": {"accessKey": "12345", "accessKey":"12345"}}`, false},
-		{`{"version" : "13", "notify": {"amqp": {"1"}, "webhook":{"3"}}}`, true},
-		{`{"version" : "13", "notify": {"amqp": {"1"}, "amqp":{"3"}}}`, false},
-		{`{"version" : "13", "notify": {"amqp": {"1":{}, "2":{}}}}`, true},
-		{`{"version" : "13", "notify": {"amqp": {"1":{}, "1":{}}}}`, false},
-	}
-
-	for i, testCase := range testCases {
-		err := doCheckDupJSONKeys(gjson.Result{}, gjson.Parse(testCase.json))
-		if testCase.shouldPass && err != nil {
-			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, err)
-		}
-		if !testCase.shouldPass && err == nil {
-			t.Errorf("Test %d, should fail but it succeed.", i+1)
-		}
-	}
-
-}
-
 // Tests config validator..
 func TestValidateConfig(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)

cmd/config-migrate.go

@@ -21,7 +21,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
 	xnet "github.com/minio/minio/pkg/net"
@@ -29,7 +31,7 @@ import (
 )
 
 // DO NOT EDIT following message template, please open a github issue to discuss instead.
-var configMigrateMSGTemplate = "Configuration file %s migrated from version '%s' to '%s' successfully.\n"
+var configMigrateMSGTemplate = "Configuration file %s migrated from version '%s' to '%s' successfully."
 
 // Migrates all config versions from "1" to "18".
 func migrateConfig() error {
@@ -41,7 +43,7 @@ func migrateConfig() error {
 	}
 
 	// Load only config version information.
-	version, err := quick.GetVersion(getConfigFile())
+	version, err := GetVersion(getConfigFile())
 	if err != nil {
 		return err
 	}
@@ -171,6 +173,21 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
+	case "23":
+		if err = migrateV23ToV24(); err != nil {
+			return err
+		}
+		fallthrough
+	case "24":
+		if err = migrateV24ToV25(); err != nil {
+			return err
+		}
+		fallthrough
+	case "25":
+		if err = migrateV25ToV26(); err != nil {
+			return err
+		}
+		fallthrough
 	case serverConfigVersion:
 		// No migration needed. this always points to current version.
 		err = nil
@@ -183,8 +200,8 @@ func purgeV1() error {
 	configFile := filepath.Join(getConfigDir(), "fsUsers.json")
 
 	cv1 := &configV1{}
-	_, err := quick.Load(configFile, cv1)
-	if os.IsNotExist(err) {
+	_, err := Load(configFile, cv1)
+	if os.IsNotExist(err) || err == dns.ErrNoEntriesFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
@@ -194,7 +211,7 @@ func purgeV1() error {
 	}
 
 	os.RemoveAll(configFile)
-	log.Println("Removed unsupported config version ‘1’.")
+	logger.Info("Removed unsupported config version ‘1’.")
 	return nil
 }
 
@@ -204,7 +221,7 @@ func migrateV2ToV3() error {
 	configFile := getConfigFile()
 
 	cv2 := &configV2{}
-	_, err := quick.Load(configFile, cv2)
+	_, err := Load(configFile, cv2)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -248,11 +265,11 @@ func migrateV2ToV3() error {
 	}
 	srvConfig.Logger.Syslog = slogger
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv2.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv2.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv2.Version, srvConfig.Version)
 	return nil
 }
 
@@ -263,7 +280,7 @@ func migrateV3ToV4() error {
 	configFile := getConfigFile()
 
 	cv3 := &configV3{}
-	_, err := quick.Load(configFile, cv3)
+	_, err := Load(configFile, cv3)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -286,11 +303,11 @@ func migrateV3ToV4() error {
 	srvConfig.Logger.File = cv3.Logger.File
 	srvConfig.Logger.Syslog = cv3.Logger.Syslog
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv3.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv3.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv3.Version, srvConfig.Version)
 	return nil
 }
 
@@ -301,7 +318,7 @@ func migrateV4ToV5() error {
 	configFile := getConfigFile()
 
 	cv4 := &configV4{}
-	_, err := quick.Load(configFile, cv4)
+	_, err := Load(configFile, cv4)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -327,11 +344,11 @@ func migrateV4ToV5() error {
 	srvConfig.Logger.ElasticSearch.Enable = false
 	srvConfig.Logger.Redis.Enable = false
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv4.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv4.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv4.Version, srvConfig.Version)
 	return nil
 }
 
@@ -342,7 +359,7 @@ func migrateV5ToV6() error {
 	configFile := getConfigFile()
 
 	cv5 := &configV5{}
-	_, err := quick.Load(configFile, cv5)
+	_, err := Load(configFile, cv5)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -416,11 +433,11 @@ func migrateV5ToV6() error {
 		}
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv5.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv5.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv5.Version, srvConfig.Version)
 	return nil
 }
 
@@ -431,7 +448,7 @@ func migrateV6ToV7() error {
 	configFile := getConfigFile()
 
 	cv6 := &configV6{}
-	_, err := quick.Load(configFile, cv6)
+	_, err := Load(configFile, cv6)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -472,11 +489,11 @@ func migrateV6ToV7() error {
 		srvConfig.Notify.Redis = cv6.Notify.Redis
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv6.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv6.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv6.Version, srvConfig.Version)
 	return nil
 }
 
@@ -487,7 +504,7 @@ func migrateV7ToV8() error {
 	configFile := getConfigFile()
 
 	cv7 := &serverConfigV7{}
-	_, err := quick.Load(configFile, cv7)
+	_, err := Load(configFile, cv7)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -535,11 +552,11 @@ func migrateV7ToV8() error {
 		srvConfig.Notify.Redis = cv7.Notify.Redis
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv7.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv7.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv7.Version, srvConfig.Version)
 	return nil
 }
 
@@ -549,7 +566,7 @@ func migrateV8ToV9() error {
 	configFile := getConfigFile()
 
 	cv8 := &serverConfigV8{}
-	_, err := quick.Load(configFile, cv8)
+	_, err := Load(configFile, cv8)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -605,11 +622,11 @@ func migrateV8ToV9() error {
 		srvConfig.Notify.PostgreSQL = cv8.Notify.PostgreSQL
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv8.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv8.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv8.Version, srvConfig.Version)
 	return nil
 }
 
@@ -619,7 +636,7 @@ func migrateV9ToV10() error {
 	configFile := getConfigFile()
 
 	cv9 := &serverConfigV9{}
-	_, err := quick.Load(configFile, cv9)
+	_, err := Load(configFile, cv9)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -673,11 +690,11 @@ func migrateV9ToV10() error {
 		srvConfig.Notify.PostgreSQL = cv9.Notify.PostgreSQL
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv9.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv9.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv9.Version, srvConfig.Version)
 	return nil
 }
 
@@ -687,7 +704,7 @@ func migrateV10ToV11() error {
 	configFile := getConfigFile()
 
 	cv10 := &serverConfigV10{}
-	_, err := quick.Load(configFile, cv10)
+	_, err := Load(configFile, cv10)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -744,11 +761,11 @@ func migrateV10ToV11() error {
 	srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
 	srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv10.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv10.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv10.Version, srvConfig.Version)
 	return nil
 }
 
@@ -758,7 +775,7 @@ func migrateV11ToV12() error {
 	configFile := getConfigFile()
 
 	cv11 := &serverConfigV11{}
-	_, err := quick.Load(configFile, cv11)
+	_, err := Load(configFile, cv11)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -842,11 +859,11 @@ func migrateV11ToV12() error {
 		}
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv11.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv11.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv11.Version, srvConfig.Version)
 	return nil
 }
 
@@ -855,7 +872,7 @@ func migrateV12ToV13() error {
 	configFile := getConfigFile()
 
 	cv12 := &serverConfigV12{}
-	_, err := quick.Load(configFile, cv12)
+	_, err := Load(configFile, cv12)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -922,11 +939,11 @@ func migrateV12ToV13() error {
 	srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
 	srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv12.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv12.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv12.Version, srvConfig.Version)
 	return nil
 }
 
@@ -935,7 +952,7 @@ func migrateV13ToV14() error {
 	configFile := getConfigFile()
 
 	cv13 := &serverConfigV13{}
-	_, err := quick.Load(configFile, cv13)
+	_, err := Load(configFile, cv13)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1007,11 +1024,11 @@ func migrateV13ToV14() error {
 	// Set the new browser parameter to true by default
 	srvConfig.Browser = true
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv13.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv13.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv13.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1020,7 +1037,7 @@ func migrateV14ToV15() error {
 	configFile := getConfigFile()
 
 	cv14 := &serverConfigV14{}
-	_, err := quick.Load(configFile, cv14)
+	_, err := Load(configFile, cv14)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1096,11 +1113,11 @@ func migrateV14ToV15() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv14.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv14.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv14.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv14.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1110,7 +1127,7 @@ func migrateV15ToV16() error {
 	configFile := getConfigFile()
 
 	cv15 := &serverConfigV15{}
-	_, err := quick.Load(configFile, cv15)
+	_, err := Load(configFile, cv15)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1186,11 +1203,11 @@ func migrateV15ToV16() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv15.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv15.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv15.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1200,7 +1217,7 @@ func migrateV16ToV17() error {
 	configFile := getConfigFile()
 
 	cv16 := &serverConfigV16{}
-	_, err := quick.Load(configFile, cv16)
+	_, err := Load(configFile, cv16)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1307,11 +1324,11 @@ func migrateV16ToV17() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv16.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv16.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv16.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv16.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1321,7 +1338,7 @@ func migrateV17ToV18() error {
 	configFile := getConfigFile()
 
 	cv17 := &serverConfigV17{}
-	_, err := quick.Load(configFile, cv17)
+	_, err := Load(configFile, cv17)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1411,11 +1428,11 @@ func migrateV17ToV18() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv17.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv17.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv17.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv17.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1423,7 +1440,7 @@ func migrateV18ToV19() error {
 	configFile := getConfigFile()
 
 	cv18 := &serverConfigV18{}
-	_, err := quick.Load(configFile, cv18)
+	_, err := Load(configFile, cv18)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1517,11 +1534,11 @@ func migrateV18ToV19() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv18.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv18.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1529,7 +1546,7 @@ func migrateV19ToV20() error {
 	configFile := getConfigFile()
 
 	cv19 := &serverConfigV19{}
-	_, err := quick.Load(configFile, cv19)
+	_, err := Load(configFile, cv19)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1622,11 +1639,11 @@ func migrateV19ToV20() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv19.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1634,7 +1651,7 @@ func migrateV20ToV21() error {
 	configFile := getConfigFile()
 
 	cv20 := &serverConfigV20{}
-	_, err := quick.Load(configFile, cv20)
+	_, err := Load(configFile, cv20)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1726,11 +1743,11 @@ func migrateV20ToV21() error {
 	// Load domain config from existing config in the file.
 	srvConfig.Domain = cv20.Domain
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv20.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv20.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv20.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1738,7 +1755,7 @@ func migrateV21ToV22() error {
 	configFile := getConfigFile()
 
 	cv21 := &serverConfigV21{}
-	_, err := quick.Load(configFile, cv21)
+	_, err := Load(configFile, cv21)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1830,11 +1847,11 @@ func migrateV21ToV22() error {
 	// Load domain config from existing config in the file.
 	srvConfig.Domain = cv21.Domain
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv21.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv21.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv21.Version, srvConfig.Version)
 	return nil
 }
 
@@ -1842,7 +1859,7 @@ func migrateV22ToV23() error {
 	configFile := getConfigFile()
 
 	cv22 := &serverConfigV22{}
-	_, err := quick.Load(configFile, cv22)
+	_, err := Load(configFile, cv22)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1943,10 +1960,360 @@ func migrateV22ToV23() error {
 	srvConfig.Cache.Exclude = []string{}
 	srvConfig.Cache.Expiry = globalCacheExpiry
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv22.Version, srvConfig.Version, err)
 	}
 
-	log.Printf(configMigrateMSGTemplate, configFile, cv22.Version, srvConfig.Version)
+	logger.Info(configMigrateMSGTemplate, configFile, cv22.Version, srvConfig.Version)
+	return nil
+}
+
+func migrateV23ToV24() error {
+	configFile := getConfigFile()
+
+	cv23 := &serverConfigV23{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv23)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘23’. %v", err)
+	}
+	if cv23.Version != "23" {
+		return nil
+	}
+
+	// Copy over fields from V23 into V24 config struct
+	srvConfig := &serverConfigV24{
+		Notify: notifier{},
+	}
+	srvConfig.Version = "24"
+	srvConfig.Credential = cv23.Credential
+	srvConfig.Region = cv23.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	if len(cv23.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]target.AMQPArgs)
+		srvConfig.Notify.AMQP["1"] = target.AMQPArgs{}
+	} else {
+		srvConfig.Notify.AMQP = cv23.Notify.AMQP
+	}
+	if len(cv23.Notify.Elasticsearch) == 0 {
+		srvConfig.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
+		srvConfig.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Elasticsearch = cv23.Notify.Elasticsearch
+	}
+	if len(cv23.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]target.RedisArgs)
+		srvConfig.Notify.Redis["1"] = target.RedisArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv23.Notify.Redis
+	}
+	if len(cv23.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
+		srvConfig.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv23.Notify.PostgreSQL
+	}
+	if len(cv23.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
+		srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
+	} else {
+		srvConfig.Notify.Kafka = cv23.Notify.Kafka
+	}
+	if len(cv23.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]target.NATSArgs)
+		srvConfig.Notify.NATS["1"] = target.NATSArgs{}
+	} else {
+		srvConfig.Notify.NATS = cv23.Notify.NATS
+	}
+	if len(cv23.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
+		srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
+	} else {
+		srvConfig.Notify.Webhook = cv23.Notify.Webhook
+	}
+	if len(cv23.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]target.MySQLArgs)
+		srvConfig.Notify.MySQL["1"] = target.MySQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv23.Notify.MySQL
+	}
+
+	if len(cv23.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]target.MQTTArgs)
+		srvConfig.Notify.MQTT["1"] = target.MQTTArgs{}
+	} else {
+		srvConfig.Notify.MQTT = cv23.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv23.Browser
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv23.Domain
+
+	// Load storage class config from existing storage class config in the file.
+	srvConfig.StorageClass.RRS = cv23.StorageClass.RRS
+	srvConfig.StorageClass.Standard = cv23.StorageClass.Standard
+
+	// Load cache config from existing cache config in the file.
+	srvConfig.Cache.Drives = cv23.Cache.Drives
+	srvConfig.Cache.Exclude = cv23.Cache.Exclude
+	srvConfig.Cache.Expiry = cv23.Cache.Expiry
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv23.Version, srvConfig.Version, err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, cv23.Version, srvConfig.Version)
+	return nil
+}
+
+func migrateV24ToV25() error {
+	configFile := getConfigFile()
+
+	cv24 := &serverConfigV24{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv24)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘24’. %v", err)
+	}
+	if cv24.Version != "24" {
+		return nil
+	}
+
+	// Copy over fields from V24 into V25 config struct
+	srvConfig := &serverConfigV25{
+		Notify: notifier{},
+	}
+	srvConfig.Version = "25"
+	srvConfig.Credential = cv24.Credential
+	srvConfig.Region = cv24.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	if len(cv24.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]target.AMQPArgs)
+		srvConfig.Notify.AMQP["1"] = target.AMQPArgs{}
+	} else {
+		srvConfig.Notify.AMQP = cv24.Notify.AMQP
+	}
+	if len(cv24.Notify.Elasticsearch) == 0 {
+		srvConfig.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
+		srvConfig.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Elasticsearch = cv24.Notify.Elasticsearch
+	}
+	if len(cv24.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]target.RedisArgs)
+		srvConfig.Notify.Redis["1"] = target.RedisArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv24.Notify.Redis
+	}
+	if len(cv24.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
+		srvConfig.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv24.Notify.PostgreSQL
+	}
+	if len(cv24.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
+		srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
+	} else {
+		srvConfig.Notify.Kafka = cv24.Notify.Kafka
+	}
+	if len(cv24.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]target.NATSArgs)
+		srvConfig.Notify.NATS["1"] = target.NATSArgs{}
+	} else {
+		srvConfig.Notify.NATS = cv24.Notify.NATS
+	}
+	if len(cv24.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
+		srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
+	} else {
+		srvConfig.Notify.Webhook = cv24.Notify.Webhook
+	}
+	if len(cv24.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]target.MySQLArgs)
+		srvConfig.Notify.MySQL["1"] = target.MySQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv24.Notify.MySQL
+	}
+
+	if len(cv24.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]target.MQTTArgs)
+		srvConfig.Notify.MQTT["1"] = target.MQTTArgs{}
+	} else {
+		srvConfig.Notify.MQTT = cv24.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv24.Browser
+
+	// New field should be turned-off by default.
+	srvConfig.Worm = false // cv25.Worm should be used here
+	// for the next migration from v25 to v26 to persist
+	// local config value.
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv24.Domain
+
+	// Load storage class config from existing storage class config in the file.
+	srvConfig.StorageClass.RRS = cv24.StorageClass.RRS
+	srvConfig.StorageClass.Standard = cv24.StorageClass.Standard
+
+	// Load cache config from existing cache config in the file.
+	srvConfig.Cache.Drives = cv24.Cache.Drives
+	srvConfig.Cache.Exclude = cv24.Cache.Exclude
+	srvConfig.Cache.Expiry = cv24.Cache.Expiry
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv24.Version, srvConfig.Version, err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, cv24.Version, srvConfig.Version)
+	return nil
+}
+
+func migrateV25ToV26() error {
+	configFile := getConfigFile()
+
+	cv25 := &serverConfigV25{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘25’. %v", err)
+	}
+	if cv25.Version != "25" {
+		return nil
+	}
+
+	// Copy over fields from V25 into V26 config struct
+	srvConfig := &serverConfigV26{
+		Notify: notifier{},
+	}
+	srvConfig.Version = "26"
+	srvConfig.Credential = cv25.Credential
+	srvConfig.Region = cv25.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	if len(cv25.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]target.AMQPArgs)
+		srvConfig.Notify.AMQP["1"] = target.AMQPArgs{}
+	} else {
+		srvConfig.Notify.AMQP = cv25.Notify.AMQP
+	}
+	if len(cv25.Notify.Elasticsearch) == 0 {
+		srvConfig.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
+		srvConfig.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Elasticsearch = cv25.Notify.Elasticsearch
+	}
+	if len(cv25.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]target.RedisArgs)
+		srvConfig.Notify.Redis["1"] = target.RedisArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv25.Notify.Redis
+	}
+	if len(cv25.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
+		srvConfig.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv25.Notify.PostgreSQL
+	}
+	if len(cv25.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
+		srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
+	} else {
+		srvConfig.Notify.Kafka = cv25.Notify.Kafka
+	}
+	if len(cv25.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]target.NATSArgs)
+		srvConfig.Notify.NATS["1"] = target.NATSArgs{}
+	} else {
+		srvConfig.Notify.NATS = cv25.Notify.NATS
+	}
+	if len(cv25.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
+		srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
+	} else {
+		srvConfig.Notify.Webhook = cv25.Notify.Webhook
+	}
+	if len(cv25.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]target.MySQLArgs)
+		srvConfig.Notify.MySQL["1"] = target.MySQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv25.Notify.MySQL
+	}
+
+	if len(cv25.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]target.MQTTArgs)
+		srvConfig.Notify.MQTT["1"] = target.MQTTArgs{}
+	} else {
+		srvConfig.Notify.MQTT = cv25.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv25.Browser
+
+	// Load worm config from existing config in the file.
+	srvConfig.Worm = cv25.Worm
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv25.Domain
+
+	// Load storage class config from existing storage class config in the file.
+	srvConfig.StorageClass.RRS = cv25.StorageClass.RRS
+	srvConfig.StorageClass.Standard = cv25.StorageClass.Standard
+
+	// Load cache config from existing cache config in the file.
+	srvConfig.Cache.Drives = cv25.Cache.Drives
+	srvConfig.Cache.Exclude = cv25.Cache.Exclude
+	srvConfig.Cache.Expiry = cv25.Cache.Expiry
+
+	// Add predefined value to new server config.
+	srvConfig.Cache.MaxUse = globalCacheMaxUse
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv25.Version, srvConfig.Version, err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, cv25.Version, srvConfig.Version)
 	return nil
 }

cmd/config-versions.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/quick"
 )
 
 /////////////////// Config V1 ///////////////////
@@ -405,7 +406,7 @@ type serverConfigV14 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -422,7 +423,7 @@ type serverConfigV15 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -460,7 +461,7 @@ type serverConfigV16 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -479,7 +480,7 @@ type serverConfigV17 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -498,7 +499,7 @@ type serverConfigV18 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -516,7 +517,7 @@ type serverConfigV19 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -534,7 +535,7 @@ type serverConfigV20 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Additional error logging configuration.
@@ -552,7 +553,7 @@ type serverConfigV21 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Notification queue configuration.
@@ -570,7 +571,7 @@ type serverConfigV22 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
@@ -580,14 +581,95 @@ type serverConfigV22 struct {
 	Notify notifier `json:"notify"`
 }
 
-// serverConfigV23 is just like version '22' with addition of cache field
+// serverConfigV23 is just like version '22' with addition of cache field.
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV23 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Browser    BoolFlag         `json:"browser"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// serverConfigV24 is just like version '23', we had to revert
+// the changes which were made in 6fb06045028b7a57c37c60a612c8e50735279ab4
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV24 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// serverConfigV25 is just like version '24', stores additionally
+// worm variable.
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV25 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// serverConfigV26 is just like version '25', stores additionally
+// cache max use value in 'CacheConfig'.
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV26 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration

cmd/crossdomain-xml-handler_test.go

@@ -21,14 +21,14 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 )
 
 // Test cross domain xml handler.
 func TestCrossXMLHandler(t *testing.T) {
 	// Server initialization.
-	mux := router.NewRouter().SkipClean(true)
-	handler := setCrossDomainPolicy(mux)
+	router := mux.NewRouter().SkipClean(true)
+	handler := setCrossDomainPolicy(router)
 	srv := httptest.NewServer(handler)
 
 	resp, err := http.Get(srv.URL + crossDomainXMLEntity)

cmd/crypto/doc.go

@@ -0,0 +1,116 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package crypto implements AWS S3 related cryptographic building blocks
+// for implementing Server-Side-Encryption (SSE-S3) and Server-Side-Encryption
+// with customer provided keys (SSE-C).
+//
+// All objects are encrypted with an unique and randomly generated 'ObjectKey'.
+// The ObjectKey itself is never stored in plaintext. Instead it is only stored
+// in a sealed from. The sealed 'ObjectKey' is created by encrypting the 'ObjectKey'
+// with an unique key-encryption-key. Given the correct key-encryption-key the
+// sealed 'ObjectKey' can be unsealed and the object can be decrypted.
+//
+//
+// ## SSE-C
+//
+// SSE-C computes the key-encryption-key from the client-provided key, an
+// initialization vector (IV) and the bucket/object path.
+//
+// 1. Encrypt:
+//       Input: ClientKey, bucket, object, metadata, object_data
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: ClientKey, bucket, object, metadata, enc_object_data
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -   KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+//
+// ## SSE-S3
+//
+// SSE-S3 can use either a master key or a KMS as root-of-trust.
+// The en/decryption slightly depens upon which root-of-trust is used.
+//
+// ### SSE-S3 and single master key
+//
+// The master key is used to derive unique object- and key-encryption-keys.
+// SSE-S3 with a single master key works as SSE-C where the master key is
+// used as the client-provided key.
+//
+// 1. Encrypt:
+//       Input: MasterKey, bucket, object, metadata, object_data
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: MasterKey, bucket, object, metadata, enc_object_data
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -   KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+//
+// ### SSE-S3 and KMS
+//
+// SSE-S3 requires that the KMS provides two functions:
+//       1.       Generate(KeyID) -> (Key, EncKey)
+//       2. Unseal(KeyID, EncKey) -> Key
+//
+// 1. Encrypt:
+//       Input: KeyID, bucket, object, metadata, object_data
+//       -     Key, EncKey := Generate(KeyID)
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(Key, Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- KeyID
+//       -        metadata <- EncKey
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: bucket, object, metadata, enc_object_data
+//       -      KeyID  <- metadata
+//       -      EncKey <- metadata
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -         Key := Unseal(KeyID, EncKey)
+//       -   KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+package crypto

cmd/crypto/error.go

@@ -0,0 +1,23 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import "errors"
+
+var (
+	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
+	// is not supported.
+	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+)

cmd/crypto/header.go

@@ -0,0 +1,49 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"net/http"
+)
+
+// SSEHeader is the general AWS SSE HTTP header key.
+const SSEHeader = "X-Amz-Server-Side-Encryption"
+
+// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
+// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
+// For SSE-C  see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
+const SSEAlgorithmAES256 = "AES256"
+
+// S3 represents AWS SSE-S3. It provides functionality to handle
+// SSE-S3 requests.
+var S3 = s3{}
+
+type s3 struct{}
+
+// IsRequested returns true if the HTTP headers indicates that
+// the S3 client requests SSE-S3.
+func (s3) IsRequested(h http.Header) bool {
+	_, ok := h[SSEHeader]
+	return ok
+}
+
+// Parse parses the SSE-S3 related HTTP headers and checks
+// whether they contain valid values.
+func (s3) Parse(h http.Header) (err error) {
+	if h.Get(SSEHeader) != SSEAlgorithmAES256 {
+		err = ErrInvalidEncryptionMethod
+	}
+	return
+}

cmd/crypto/header_test.go

@@ -0,0 +1,56 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"net/http"
+	"testing"
+)
+
+var isRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},  // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},        // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},  // 3
+}
+
+func TestS3IsRequested(t *testing.T) {
+	for i, test := range isRequestedTests {
+		if got := S3.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var parseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, ExpectedErr: nil},                         // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, ExpectedErr: ErrInvalidEncryptionMethod},        // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, ExpectedErr: ErrInvalidEncryptionMethod},   // 3
+}
+
+func TestS3Parse(t *testing.T) {
+	for i, test := range parseTests {
+		if err := S3.Parse(test.Header); err != test.ExpectedErr {
+			t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+	}
+}

cmd/crypto/key.go

@@ -0,0 +1,93 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"io"
+	"path/filepath"
+
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+// ObjectKey is a 256 bit secret key used to encrypt the object.
+// It must never be stored in plaintext.
+type ObjectKey [32]byte
+
+// GenerateKey generates a unique ObjectKey from a 256 bit external key
+// and a source of randomness. If random is nil the default PRNG of system
+// (crypto/rand) is used.
+func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
+	if random == nil {
+		random = rand.Reader
+	}
+	var nonce [32]byte
+	if _, err := io.ReadFull(random, nonce[:]); err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
+	}
+	sha := sha256.New()
+	sha.Write(extKey[:])
+	sha.Write(nonce[:])
+	sha.Sum(key[:0])
+	return
+}
+
+// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
+// key is also cryptographically bound to the object's path (bucket/object).
+func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte {
+	var sealedKey bytes.Buffer
+	mac := hmac.New(sha256.New, extKey[:])
+	mac.Write(iv[:])
+	mac.Write([]byte(filepath.Join(bucket, object)))
+
+	if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
+	}
+	return sealedKey.Bytes()
+}
+
+// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key
+// is cryptographically bound to the object's path the same bucket/object as during sealing
+// must be provided. On success the ObjectKey contains the decrypted sealed key.
+func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error {
+	var unsealedKey bytes.Buffer
+	mac := hmac.New(sha256.New, extKey[:])
+	mac.Write(iv[:])
+	mac.Write([]byte(filepath.Join(bucket, object)))
+
+	if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil {
+		return err // TODO(aead): upgrade sio to use sio.Error
+	}
+	copy(key[:], unsealedKey.Bytes())
+	return nil
+}
+
+// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index.
+func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
+	var bin [4]byte
+	binary.LittleEndian.PutUint32(bin[:], id)
+
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write(bin[:])
+	mac.Sum(partKey[:0])
+	return
+}

cmd/crypto/key_test.go

@@ -0,0 +1,127 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"io"
+	"testing"
+)
+
+var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
+
+func recoverTest(i int, shouldPass bool, t *testing.T) {
+	if err := recover(); err == nil && !shouldPass {
+		t.Errorf("Test %d should fail but passed successfully", i)
+	} else if err != nil && shouldPass {
+		t.Errorf("Test %d should pass but failed: %v", i, err)
+	}
+}
+
+var generateKeyTests = []struct {
+	ExtKey     [32]byte
+	Random     io.Reader
+	ShouldPass bool
+}{
+	{ExtKey: [32]byte{}, Random: nil, ShouldPass: true},             // 0
+	{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true},     // 1
+	{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
+	// {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064
+}
+
+func TestGenerateKey(t *testing.T) {
+	for i, test := range generateKeyTests {
+		func() {
+			defer recoverTest(i, test.ShouldPass, t)
+			key := GenerateKey(test.ExtKey, test.Random)
+			if [32]byte(key) == [32]byte{} {
+				t.Errorf("Test %d: generated key is zero key", i) // check that we generate random and unique key
+			}
+		}()
+	}
+}
+
+var sealUnsealKeyTests = []struct {
+	SealExtKey, SealIV     [32]byte
+	SealBucket, SealObject string
+
+	UnsealExtKey, UnsealIV     [32]byte
+	UnsealBucket, UnsealObject string
+
+	ShouldPass bool
+}{
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: true,
+	}, // 0
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 1
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 2
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 3
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object",
+		ShouldPass: false,
+	}, // 4
+}
+
+func TestSealUnsealKey(t *testing.T) {
+	for i, test := range sealUnsealKeyTests {
+		key := GenerateKey(test.SealExtKey, rand.Reader)
+		sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject)
+		if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
+			t.Errorf("Test %d should fail but passed successfully", i)
+		} else if err != nil && test.ShouldPass {
+			t.Errorf("Test %d should pass put failed: %v", i, err)
+		}
+	}
+}
+
+var derivePartKeyTest = []struct {
+	PartID  uint32
+	PartKey string
+}{
+	{PartID: 0, PartKey: "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"},
+	{PartID: 1, PartKey: "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"},
+	{PartID: 10000, PartKey: "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"},
+}
+
+func TestDerivePartKey(t *testing.T) {
+	var key ObjectKey
+	for i, test := range derivePartKeyTest {
+		expectedPartKey, err := hex.DecodeString(test.PartKey)
+		if err != nil {
+			t.Fatalf("Test %d failed to decode expected part-key: %v", i, err)
+		}
+		partKey := key.DerivePartKey(test.PartID)
+		if !bytes.Equal(partKey[:], expectedPartKey[:]) {
+			t.Errorf("Test %d derives wrong part-key: got '%s' want: '%s'", i, hex.EncodeToString(partKey[:]), test.PartKey)
+		}
+	}
+}

cmd/crypto/sse.go

@@ -0,0 +1,60 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"context"
+	"errors"
+	"io"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/ioutil"
+	"github.com/minio/sio"
+)
+
+const (
+	// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
+	S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
+	// S3KMSKeyID is the metadata key referencing the KMS key-id used to
+	// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
+	S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
+	// S3KMSSealedKey is the metadata key referencing the encrypted key generated
+	// by KMS. It is only used for SSE-S3 + KMS.
+	S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
+)
+
+// EncryptSinglePart encrypts an io.Reader which must be the
+// the body of a single-part PUT request.
+func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
+	r, err := sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, Key: key[:]})
+	if err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to encrypt io.Reader using object key"))
+	}
+	return r
+}
+
+// DecryptSinglePart decrypts an io.Writer which must an object
+// uploaded with the single-part PUT API. The offset and length
+// specify the requested range.
+func DecryptSinglePart(w io.Writer, offset, length int64, key ObjectKey) io.WriteCloser {
+	const PayloadSize = 1 << 16 // DARE 2.0
+	w = ioutil.LimitedWriter(w, offset%PayloadSize, length)
+
+	decWriter, err := sio.DecryptWriter(w, sio.Config{Key: key[:]})
+	if err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to decrypt io.Writer using object key"))
+	}
+	return decWriter
+}

cmd/disk-cache-config.go

@@ -18,14 +18,17 @@ package cmd
 
 import (
 	"encoding/json"
-	"fmt"
 	"path/filepath"
+	"strings"
+
+	"github.com/minio/minio/pkg/ellipses"
 )
 
 // CacheConfig represents cache config settings
 type CacheConfig struct {
 	Drives  []string `json:"drives"`
 	Expiry  int      `json:"expiry"`
+	MaxUse  int      `json:"maxuse"`
 	Exclude []string `json:"exclude"`
 }
 
@@ -52,22 +55,52 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
 
 // Parses given cacheDrivesEnv and returns a list of cache drives.
 func parseCacheDrives(drives []string) ([]string, error) {
+	if len(drives) == 0 {
+		return drives, nil
+	}
+	var endpoints []string
 	for _, d := range drives {
-		if !filepath.IsAbs(d) {
-			return nil, fmt.Errorf("cache dir should be absolute path: %s", d)
+		if ellipses.HasEllipses(d) {
+			s, err := parseCacheDrivePaths(d)
+			if err != nil {
+				return nil, err
+			}
+			endpoints = append(endpoints, s...)
+		} else {
+			endpoints = append(endpoints, d)
 		}
 	}
-	return drives, nil
+
+	for _, d := range endpoints {
+		if !filepath.IsAbs(d) {
+			return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
+		}
+	}
+	return endpoints, nil
+}
+
+// Parses all arguments and returns a slice of drive paths following the ellipses pattern.
+func parseCacheDrivePaths(arg string) (ep []string, err error) {
+	patterns, perr := ellipses.FindEllipsesPatterns(arg)
+	if perr != nil {
+		return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
+	}
+
+	for _, lbls := range patterns.Expand() {
+		ep = append(ep, strings.Join(lbls, ""))
+	}
+
+	return ep, nil
 }
 
 // Parses given cacheExcludesEnv and returns a list of cache exclude patterns.
 func parseCacheExcludes(excludes []string) ([]string, error) {
 	for _, e := range excludes {
 		if len(e) == 0 {
-			return nil, fmt.Errorf("cache exclude path (%s) cannot be empty", e)
+			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
 		}
 		if hasPrefix(e, slashSeparator) {
-			return nil, fmt.Errorf("cache exclude pattern (%s) cannot start with / as prefix", e)
+			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
 		}
 	}
 	return excludes, nil

cmd/disk-cache-config_test.go

@@ -41,12 +41,32 @@ func TestParseCacheDrives(t *testing.T) {
 			expectedPatterns []string
 			success          bool
 		}{"C:/home/drive1;C:/home/drive2;C:/home/drive3", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1...3}", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1..3}", []string{}, false})
 	} else {
 		testCases = append(testCases, struct {
 			driveStr         string
 			expectedPatterns []string
 			success          bool
 		}{"/home/drive1;/home/drive2;/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1...3}", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1..3}", []string{}, false})
 	}
 	for i, testCase := range testCases {
 		drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheEnvDelimiter))

cmd/disk-cache-fs.go

@@ -28,8 +28,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
-	errors2 "github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 )
@@ -37,7 +37,6 @@ import (
 const (
 	// cache.json object metadata for cached objects.
 	cacheMetaJSONFile = "cache.json"
-	cacheMetaFormat   = "cache"
 
 	cacheEnvDelimiter = ";"
 )
@@ -93,7 +92,7 @@ func newCacheFSObjects(dir string, expiry int, maxDiskUsagePct int) (*cacheFSObj
 		appendFileMap: make(map[string]*fsAppendFile),
 	}
 
-	go fsObjects.cleanupStaleMultipartUploads(globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
+	go fsObjects.cleanupStaleMultipartUploads(context.Background(), globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
 
 	cacheFS := cacheFSObjects{
 		FSObjects:       fsObjects,
@@ -116,7 +115,9 @@ func (cfs *cacheFSObjects) diskUsageLow() bool {
 	minUsage := cfs.maxDiskUsagePct * 80 / 100
 	di, err := disk.GetInfo(cfs.dir)
 	if err != nil {
-		errorIf(err, "Error getting disk information on %s", cfs.dir)
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
@@ -128,7 +129,9 @@ func (cfs *cacheFSObjects) diskUsageLow() bool {
 func (cfs *cacheFSObjects) diskUsageHigh() bool {
 	di, err := disk.GetInfo(cfs.dir)
 	if err != nil {
-		errorIf(err, "Error getting disk information on %s", cfs.dir)
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
 		return true
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
@@ -140,7 +143,9 @@ func (cfs *cacheFSObjects) diskUsageHigh() bool {
 func (cfs *cacheFSObjects) diskAvailable(size int64) bool {
 	di, err := disk.GetInfo(cfs.dir)
 	if err != nil {
-		errorIf(err, "Error getting disk information on %s", cfs.dir)
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
 		return false
 	}
 	usedPercent := (di.Total - (di.Free - uint64(size))) * 100 / di.Total
@@ -150,11 +155,11 @@ func (cfs *cacheFSObjects) diskAvailable(size int64) bool {
 // purges all content marked trash from the cache.
 func (cfs *cacheFSObjects) purgeTrash() {
 	ticker := time.NewTicker(time.Minute * cacheCleanupInterval)
+	defer ticker.Stop()
+
 	for {
 		select {
 		case <-globalServiceDoneCh:
-			// Stop the timer.
-			ticker.Stop()
 			return
 		case <-ticker.C:
 			trashPath := path.Join(cfs.fsPath, minioMetaBucket, cacheTrashDir)
@@ -163,14 +168,15 @@ func (cfs *cacheFSObjects) purgeTrash() {
 				return
 			}
 			for _, entry := range entries {
-				fi, err := fsStatVolume(pathJoin(trashPath, entry))
+				ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
+				fi, err := fsStatVolume(ctx, pathJoin(trashPath, entry))
 				if err != nil {
 					continue
 				}
 				dir := path.Join(trashPath, fi.Name())
 
 				// Delete all expired cache content.
-				fsRemoveAll(dir)
+				fsRemoveAll(ctx, dir)
 			}
 		}
 	}
@@ -193,7 +199,7 @@ func (cfs *cacheFSObjects) purge() {
 			deletedCount := 0
 			buckets, err := cfs.ListBuckets(ctx)
 			if err != nil {
-				errorIf(err, "Unable to list buckets.")
+				logger.LogIf(ctx, err)
 			}
 			// Reset cache online status if drive was offline earlier.
 			if !cfs.IsOnline() {
@@ -221,7 +227,7 @@ func (cfs *cacheFSObjects) purge() {
 							continue
 						}
 						if err = cfs.DeleteObject(ctx, bucket.Name, object.Name); err != nil {
-							errorIf(err, "Unable to remove cache entry in dir %s/%s", bucket.Name, object.Name)
+							logger.LogIf(ctx, err)
 							continue
 						}
 						deletedCount++
@@ -271,7 +277,7 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data
 	}
 	_, err := cfs.PutObject(ctx, bucket, object, data, metadata)
 	// if err is due to disk being offline , mark cache drive as offline
-	if errors2.IsErr(err, baseErrs...) {
+	if IsErr(err, baseErrs...) {
 		cfs.setOnline(false)
 	}
 	return err
@@ -313,7 +319,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	var err error
 
 	// Validate if bucket name is valid and exists.
-	if _, err = fs.statBucketDir(bucket); err != nil {
+	if _, err = fs.statBucketDir(ctx, bucket); err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket)
 	}
 
@@ -325,31 +331,32 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	// and return success.
 	if isObjectDir(object, data.Size()) {
 		// Check if an object is present as one of the parent dir.
-		if fs.parentDirIsObject(bucket, path.Dir(object)) {
-			return ObjectInfo{}, toObjectErr(errors2.Trace(errFileAccessDenied), bucket, object)
+		if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
+			return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
 		}
 		if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 		var fi os.FileInfo
-		if fi, err = fsStatDir(pathJoin(fs.fsPath, bucket, object)); err != nil {
+		if fi, err = fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object)); err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 		return fsMeta.ToObjectInfo(bucket, object, fi), nil
 	}
 
-	if err = checkPutObjectArgs(bucket, object, fs, data.Size()); err != nil {
+	if err = checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil {
 		return ObjectInfo{}, err
 	}
 
 	// Check if an object is present as one of the parent dir.
-	if fs.parentDirIsObject(bucket, path.Dir(object)) {
-		return ObjectInfo{}, toObjectErr(errors2.Trace(errFileAccessDenied), bucket, object)
+	if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
+		return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
 	}
 
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < 0 {
-		return ObjectInfo{}, errors2.Trace(errInvalidArgument)
+		logger.LogIf(ctx, errInvalidArgument)
+		return ObjectInfo{}, errInvalidArgument
 	}
 
 	var wlk *lock.LockedFile
@@ -359,7 +366,8 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 
 		wlk, err = fs.rwPool.Create(fsMetaPath)
 		if err != nil {
-			return ObjectInfo{}, toObjectErr(errors2.Trace(err), bucket, object)
+			logger.LogIf(ctx, err)
+			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 		// This close will allow for locks to be synchronized on `fs.json`.
 		defer wlk.Close()
@@ -367,7 +375,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 			// Remove meta file when PutObject encounters any error
 			if retErr != nil {
 				tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)
-				fsRemoveMeta(bucketMetaDir, fsMetaPath, tmpDir)
+				fsRemoveMeta(ctx, bucketMetaDir, fsMetaPath, tmpDir)
 			}
 		}()
 	}
@@ -385,10 +393,9 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 
 	buf := make([]byte, int(bufSize))
 	fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj)
-	bytesWritten, err := fsCreateFile(fsTmpObjPath, data, buf, data.Size())
+	bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, buf, data.Size())
 	if err != nil {
-		fsRemoveFile(fsTmpObjPath)
-		errorIf(err, "Failed to create object %s/%s", bucket, object)
+		fsRemoveFile(ctx, fsTmpObjPath)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 	if fsMeta.Meta["etag"] == "" {
@@ -397,18 +404,18 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.
 	if bytesWritten < data.Size() {
-		fsRemoveFile(fsTmpObjPath)
-		return ObjectInfo{}, errors2.Trace(IncompleteBody{})
+		fsRemoveFile(ctx, fsTmpObjPath)
+		return ObjectInfo{}, IncompleteBody{}
 	}
 
 	// Delete the temporary object in the case of a
 	// failure. If PutObject succeeds, then there would be
 	// nothing to delete.
-	defer fsRemoveFile(fsTmpObjPath)
+	defer fsRemoveFile(ctx, fsTmpObjPath)
 
 	// Entire object was written to the temp location, now it's safe to rename it to the actual location.
 	fsNSObjPath := pathJoin(fs.fsPath, bucket, object)
-	if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil {
+	if err = fsRenameFile(ctx, fsTmpObjPath, fsNSObjPath); err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
@@ -420,7 +427,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	}
 
 	// Stat the file to fetch timestamp, size.
-	fi, err := fsStatFile(pathJoin(fs.fsPath, bucket, object))
+	fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
 	if err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
@@ -447,11 +454,11 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec
 		}
 	}
 	fs := cfs.FSObjects
-	if err := checkNewMultipartArgs(bucket, object, fs); err != nil {
+	if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil {
 		return "", toObjectErr(err, bucket)
 	}
 
-	if _, err := fs.statBucketDir(bucket); err != nil {
+	if _, err := fs.statBucketDir(ctx, bucket); err != nil {
 		return "", toObjectErr(err, bucket)
 	}
 
@@ -459,7 +466,8 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec
 
 	err := mkdirAll(uploadIDDir, 0755)
 	if err != nil {
-		return "", errors2.Trace(err)
+		logger.LogIf(ctx, err)
+		return "", err
 	}
 
 	// Initialize fs.json values.
@@ -468,11 +476,13 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec
 
 	fsMetaBytes, err := json.Marshal(fsMeta)
 	if err != nil {
-		return "", errors2.Trace(err)
+		logger.LogIf(ctx, err)
+		return "", err
 	}
 
 	if err = ioutil.WriteFile(pathJoin(uploadIDDir, fs.metaJSONFile), fsMetaBytes, 0644); err != nil {
-		return "", errors2.Trace(err)
+		logger.LogIf(ctx, err)
+		return "", err
 	}
 	return uploadID, nil
 }
@@ -485,7 +495,7 @@ func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string)
 		return err
 	}
 	defer bucketLock.Unlock()
-	bucketDir, err := fs.getBucketDir(bucket)
+	bucketDir, err := fs.getBucketDir(ctx, bucket)
 	if err != nil {
 		return toObjectErr(err, bucket)
 	}
@@ -493,12 +503,13 @@ func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string)
 	expiredDir := path.Join(trashPath, bucket)
 	// Attempt to move regular bucket to expired directory.
 	if err = fsRenameDir(bucketDir, expiredDir); err != nil {
+		logger.LogIf(ctx, err)
 		return toObjectErr(err, bucket)
 	}
 	// Cleanup all the bucket metadata.
 	ominioMetadataBucketDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket)
 	nminioMetadataBucketDir := pathJoin(trashPath, MustGetUUID())
-	_ = fsRenameDir(ominioMetadataBucketDir, nminioMetadataBucketDir)
+	logger.LogIf(ctx, fsRenameDir(ominioMetadataBucketDir, nminioMetadataBucketDir))
 	return nil
 }
 
@@ -506,22 +517,22 @@ func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string)
 // paths for windows automatically.
 func fsRenameDir(dirPath, newPath string) (err error) {
 	if dirPath == "" || newPath == "" {
-		return errors2.Trace(errInvalidArgument)
+		return errInvalidArgument
 	}
 
 	if err = checkPathLength(dirPath); err != nil {
-		return errors2.Trace(err)
+		return err
 	}
 	if err = checkPathLength(newPath); err != nil {
-		return errors2.Trace(err)
+		return err
 	}
 	if err = os.Rename(dirPath, newPath); err != nil {
 		if os.IsNotExist(err) {
-			return errors2.Trace(errVolumeNotFound)
+			return errVolumeNotFound
 		} else if isSysErrNotEmpty(err) {
-			return errors2.Trace(errVolumeNotEmpty)
+			return errVolumeNotEmpty
 		}
-		return errors2.Trace(err)
+		return err
 	}
 	return nil
 }

cmd/disk-cache.go

@@ -31,7 +31,7 @@ import (
 
 	"github.com/djherbis/atime"
 
-	errors2 "github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/wildcard"
 
 	"github.com/minio/minio/pkg/hash"
@@ -44,7 +44,6 @@ const (
 	// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
 	cacheSizeMultiplier  = 100
 	cacheTrashDir        = "trash"
-	cacheMaxDiskUsagePct = 80 // in %
 	cacheCleanupInterval = 10 // in minutes
 )
 
@@ -77,6 +76,13 @@ type cacheObjects struct {
 	DeleteBucketFn            func(ctx context.Context, bucket string) error
 }
 
+// CacheStorageInfo - represents total, free capacity of
+// underlying cache storage.
+type CacheStorageInfo struct {
+	Total uint64 // Total cache disk space.
+	Free  uint64 // Free cache available space.
+}
+
 // CacheObjectLayer implements primitives for cache object API layer.
 type CacheObjectLayer interface {
 	// Bucket operations.
@@ -98,13 +104,13 @@ type CacheObjectLayer interface {
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
 
 	// Storage operations.
-	StorageInfo(ctx context.Context) StorageInfo
+	StorageInfo(ctx context.Context) CacheStorageInfo
 }
 
 // backendDownError returns true if err is due to backend failure or faulty disk if in server mode
 func backendDownError(err error) bool {
-	_, backendDown := errors2.Cause(err).(BackendDown)
-	return backendDown || errors2.IsErr(err, baseErrs...)
+	_, backendDown := err.(BackendDown)
+	return backendDown || IsErr(err, baseErrs...)
 }
 
 // get cache disk where object is currently cached for a GET operation. If object does not exist at that location,
@@ -191,7 +197,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 	objInfo, err := GetObjectInfoFn(ctx, bucket, object)
 	backendDown := backendDownError(err)
 	if err != nil && !backendDown {
-		if _, ok := errors2.Cause(err).(ObjectNotFound); ok {
+		if _, ok := err.(ObjectNotFound); ok {
 			// Delete the cached entry if backend object was deleted.
 			dcache.Delete(ctx, bucket, object)
 		}
@@ -232,7 +238,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 			pipeWriter.CloseWithError(err)
 			return
 		}
-		pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
+		pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
 	}()
 	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo))
 	if err != nil {
@@ -255,7 +261,7 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string)
 	}
 	objInfo, err := getObjectInfoFn(ctx, bucket, object)
 	if err != nil {
-		if _, ok := errors2.Cause(err).(ObjectNotFound); ok {
+		if _, ok := err.(ObjectNotFound); ok {
 			// Delete the cached entry if backend object was deleted.
 			dcache.Delete(ctx, bucket, object)
 			return ObjectInfo{}, err
@@ -289,12 +295,20 @@ func listDirCacheFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks [
 	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string, err error) {
 		var entries []string
 		for _, disk := range disks {
+			// ignore disk-caches that might be missing/offline
+			if disk == nil {
+				continue
+			}
 			fs := disk.FSObjects
 			entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
+
+			// For any reason disk was deleted or goes offline, continue
+			// and list from other disks if possible.
 			if err != nil {
-				// For any reason disk was deleted or goes offline, continue
-				// and list from other disks if possible.
-				continue
+				if IsErrIgnored(err, treeWalkIgnoredErrs...) {
+					continue
+				}
+				return nil, err
 			}
 
 			// Filter entries that have the prefix prefixEntry.
@@ -345,12 +359,20 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			if err != nil {
 				return false
 			}
-			_, err = fs.getObjectInfo(bucket, object)
+			_, err = fs.getObjectInfo(ctx, bucket, object)
 			return err == nil
 		}
 
+		isLeafDir := func(bucket, object string) bool {
+			fs, err := c.cache.getCacheFS(ctx, bucket, object)
+			if err != nil {
+				return false
+			}
+			return fs.isObjectDir(bucket, object)
+		}
+
 		listDir := listDirCacheFactory(isLeaf, cacheTreeWalkIgnoredErrs, c.cache.cfs)
-		walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh)
+		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 
 	for i := 0; i < maxKeys; {
@@ -377,16 +399,16 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			var err error
 			fs, err := c.cache.getCacheFS(ctx, bucket, entry)
 			if err != nil {
-				// Ignore errFileNotFound
-				if errors2.Cause(err) == errFileNotFound {
+				// Ignore errDiskNotFound
+				if err == errDiskNotFound {
 					continue
 				}
 				return result, toObjectErr(err, bucket, prefix)
 			}
-			objInfo, err = fs.getObjectInfo(bucket, entry)
+			objInfo, err = fs.getObjectInfo(ctx, bucket, entry)
 			if err != nil {
-				// Ignore errFileNotFound
-				if errors2.Cause(err) == errFileNotFound {
+				// Ignore ObjectNotFound error
+				if _, ok := err.(ObjectNotFound); ok {
 					continue
 				}
 				return result, toObjectErr(err, bucket, prefix)
@@ -409,7 +431,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 	result = ListObjectsInfo{IsTruncated: !eof}
 	for _, objInfo := range objInfos {
 		result.NextMarker = objInfo.Name
-		if objInfo.IsDir {
+		if objInfo.IsDir && delimiter == slashSeparator {
 			result.Prefixes = append(result.Prefixes, objInfo.Name)
 			continue
 		}
@@ -469,6 +491,10 @@ func (c cacheObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continu
 func (c cacheObjects) listBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
 	m := make(map[string]string)
 	for _, cache := range c.cache.cfs {
+		// ignore disk-caches that might be missing/offline
+		if cache == nil {
+			continue
+		}
 		entries, err := cache.ListBuckets(ctx)
 
 		if err != nil {
@@ -507,6 +533,10 @@ func (c cacheObjects) GetBucketInfo(ctx context.Context, bucket string) (bucketI
 	bucketInfo, err = getBucketInfoFn(ctx, bucket)
 	if backendDownError(err) {
 		for _, cache := range c.cache.cfs {
+			// ignore disk-caches that might be missing/offline
+			if cache == nil {
+				continue
+			}
 			if bucketInfo, err = cache.GetBucketInfo(ctx, bucket); err == nil {
 				return
 			}
@@ -747,23 +777,22 @@ func (c cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, objec
 }
 
 // StorageInfo - returns underlying storage statistics.
-func (c cacheObjects) StorageInfo(ctx context.Context) (storageInfo StorageInfo) {
+func (c cacheObjects) StorageInfo(ctx context.Context) (cInfo CacheStorageInfo) {
 	var total, free uint64
 	for _, cfs := range c.cache.cfs {
 		if cfs == nil {
 			continue
 		}
-		info, err := getDiskInfo((cfs.fsPath))
-		errorIf(err, "Unable to get disk info %#v", cfs.fsPath)
+		info, err := getDiskInfo(cfs.fsPath)
+		logger.GetReqInfo(ctx).AppendTags("cachePath", cfs.fsPath)
+		logger.LogIf(ctx, err)
 		total += info.Total
 		free += info.Free
 	}
-	storageInfo = StorageInfo{
+	return CacheStorageInfo{
 		Total: total,
 		Free:  free,
 	}
-	storageInfo.Backend.Type = FS
-	return storageInfo
 }
 
 // DeleteBucket - marks bucket to be deleted from cache if bucket is deleted from backend.
@@ -771,6 +800,10 @@ func (c cacheObjects) DeleteBucket(ctx context.Context, bucket string) (err erro
 	deleteBucketFn := c.DeleteBucketFn
 	var toDel []*cacheFSObjects
 	for _, cfs := range c.cache.cfs {
+		// ignore disk-caches that might be missing/offline
+		if cfs == nil {
+			continue
+		}
 		if _, cerr := cfs.GetBucketInfo(ctx, bucket); cerr == nil {
 			toDel = append(toDel, cfs)
 		}
@@ -791,7 +824,8 @@ func (c cacheObjects) DeleteBucket(ctx context.Context, bucket string) (err erro
 // or the global env overrides.
 func newCache(config CacheConfig) (*diskCache, error) {
 	var cfsObjects []*cacheFSObjects
-	formats, err := loadAndValidateCacheFormat(config.Drives)
+	ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
+	formats, err := loadAndValidateCacheFormat(ctx, config.Drives)
 	if err != nil {
 		return nil, err
 	}
@@ -804,7 +838,7 @@ func newCache(config CacheConfig) (*diskCache, error) {
 		if err := checkAtimeSupport(dir); err != nil {
 			return nil, errors.New("Atime support required for disk caching")
 		}
-		cache, err := newCacheFSObjects(dir, config.Expiry, cacheMaxDiskUsagePct)
+		cache, err := newCacheFSObjects(dir, config.Expiry, config.MaxUse)
 		if err != nil {
 			return nil, err
 		}

cmd/disk-cache_test.go

@@ -28,10 +28,10 @@ import (
 )
 
 // Initialize cache FS objects.
-func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
+func initCacheFSObjects(disk string, cacheMaxUse int, t *testing.T) (*cacheFSObjects, error) {
 	newTestConfig(globalMinioDefaultRegion)
 	var err error
-	obj, err := newCacheFSObjects(disk, globalCacheExpiry, 100)
+	obj, err := newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -39,10 +39,10 @@ func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
 }
 
 // inits diskCache struct for nDisks
-func initDiskCaches(drives []string, t *testing.T) (*diskCache, error) {
+func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
 	var cfs []*cacheFSObjects
 	for _, d := range drives {
-		obj, err := initCacheFSObjects(d, t)
+		obj, err := initCacheFSObjects(d, cacheMaxUse, t)
 		if err != nil {
 			return nil, err
 		}
@@ -59,7 +59,46 @@ func TestGetCacheFS(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, t)
+		d, err := initDiskCaches(fsDirs, 100, t)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bucketName := "testbucket"
+		objectName := "testobject"
+		ctx := context.Background()
+		// find cache drive where object would be hashed
+		index := d.hashIndex(bucketName, objectName)
+		// turn off drive by setting online status to false
+		d.cfs[index].online = false
+		cfs, err := d.getCacheFS(ctx, bucketName, objectName)
+		if n == 1 && err == errDiskNotFound {
+			continue
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		i := -1
+		for j, f := range d.cfs {
+			if f == cfs {
+				i = j
+				break
+			}
+		}
+		if i != (index+1)%n {
+			t.Fatalf("expected next cache location to be picked")
+		}
+	}
+}
+
+// test whether a drive being offline causes
+// getCacheFS to fetch next online drive
+func TestGetCacheFSMaxUse(t *testing.T) {
+	for n := 1; n < 10; n++ {
+		fsDirs, err := getRandomDisks(n)
+		if err != nil {
+			t.Fatal(err)
+		}
+		d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -141,7 +180,7 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, t)
+	d, err := initDiskCaches(fsDirs, 100, t)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -209,6 +248,87 @@ func TestDiskCache(t *testing.T) {
 	}
 }
 
+// Test diskCache with upper bound on max cache use.
+func TestDiskCacheMaxUse(t *testing.T) {
+	fsDirs, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cache := d.cfs[0]
+	ctx := context.Background()
+	bucketName := "testbucket"
+	objectName := "testobject"
+	content := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	etag := "061208c10af71a30c6dcd6cf5d89f0fe"
+	contentType := "application/zip"
+	size := len(content)
+
+	httpMeta := make(map[string]string)
+	httpMeta["etag"] = etag
+	httpMeta["content-type"] = contentType
+
+	objInfo := ObjectInfo{}
+	objInfo.Bucket = bucketName
+	objInfo.Name = objectName
+	objInfo.Size = int64(size)
+	objInfo.ContentType = contentType
+	objInfo.ETag = etag
+	objInfo.UserDefined = httpMeta
+
+	byteReader := bytes.NewReader([]byte(content))
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !cache.diskAvailable(int64(size)) {
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		if err != errDiskFull {
+			t.Fatal("Cache max-use limit violated.")
+		}
+	} else {
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		if err != nil {
+			t.Fatal(err)
+		}
+		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !cache.Exists(ctx, bucketName, objectName) {
+			t.Fatal("Expected object to exist on cache")
+		}
+		if cachedObjInfo.ETag != objInfo.ETag {
+			t.Fatal("Expected ETag to match")
+		}
+		if cachedObjInfo.Size != objInfo.Size {
+			t.Fatal("Size mismatch")
+		}
+		if cachedObjInfo.ContentType != objInfo.ContentType {
+			t.Fatal("Cached content-type does not match")
+		}
+		writer := bytes.NewBuffer(nil)
+		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
+			t.Errorf("wrong cached file content")
+		}
+		err = cache.Delete(ctx, bucketName, objectName)
+		if err != nil {
+			t.Errorf("object missing from cache")
+		}
+		online := cache.IsOnline()
+		if !online {
+			t.Errorf("expected cache drive to be online")
+		}
+	}
+}
+
 func TestIsCacheExcludeDirective(t *testing.T) {
 	testCases := []struct {
 		cacheControlOpt string

cmd/disk-usage.go

@@ -0,0 +1,54 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+)
+
+// getDiskUsage walks the file tree rooted at root, calling usageFn
+// for each file or directory in the tree, including root.
+func getDiskUsage(ctx context.Context, root string, usageFn usageFunc) error {
+	return walk(ctx, root+slashSeparator, usageFn)
+}
+
+type usageFunc func(ctx context.Context, entry string) error
+
+// walk recursively descends path, calling walkFn.
+func walk(ctx context.Context, path string, usageFn usageFunc) error {
+	if err := usageFn(ctx, path); err != nil {
+		return err
+	}
+
+	if !hasSuffix(path, slashSeparator) {
+		return nil
+	}
+
+	entries, err := readDir(path)
+	if err != nil {
+		return usageFn(ctx, path)
+	}
+
+	for _, entry := range entries {
+		fname := pathJoin(path, entry)
+		if err = walk(ctx, fname, usageFn); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/dummy-object-layer_test.go

@@ -0,0 +1,165 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/policy"
+)
+
+type DummyObjectLayer struct{}
+
+func (api *DummyObjectLayer) Shutdown(context.Context) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) StorageInfo(context.Context) (si StorageInfo) {
+	return
+}
+
+func (api *DummyObjectLayer) MakeBucketWithLocation(ctx context.Context, bucket string, location string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) DeleteBucket(ctx context.Context, bucket string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) DeleteObject(ctx context.Context, bucket, object string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo) (info PartInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) HealFormat(ctx context.Context, dryRun bool) (item madmin.HealResultItem, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) HealBucket(ctx context.Context, bucket string, dryRun bool) (items []madmin.HealResultItem, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) HealObject(ctx context.Context, bucket, object string, dryRun bool) (item madmin.HealResultItem, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (info ListObjectsInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) (info []VolumeLockInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) ClearLocks(context.Context, []VolumeLockInfo) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) SetBucketPolicy(context.Context, string, *policy.Policy) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) GetBucketPolicy(context.Context, string) (bucketPolicy *policy.Policy, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) RefreshBucketPolicy(context.Context, string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) DeleteBucketPolicy(context.Context, string) (err error) {
+	return
+}
+
+func (api *DummyObjectLayer) IsNotificationSupported() (b bool) {
+	return
+}
+
+func (api *DummyObjectLayer) IsEncryptionSupported() (b bool) {
+	return
+}

cmd/encryption-v1.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/hmac"
 	"crypto/md5"
 	"crypto/rand"
@@ -27,7 +28,10 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"path"
+	"strconv"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"
 	sha256 "github.com/minio/sha256-simd"
 	"github.com/minio/sio"
@@ -35,14 +39,15 @@ import (
 
 var (
 	// AWS errors for invalid SSE-C requests.
-	errInsecureSSERequest  = errors.New("Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection")
-	errEncryptedObject     = errors.New("The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object")
-	errInvalidSSEAlgorithm = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm")
-	errMissingSSEKey       = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key")
-	errInvalidSSEKey       = errors.New("The secret key was invalid for the specified algorithm")
-	errMissingSSEKeyMD5    = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key")
-	errSSEKeyMD5Mismatch   = errors.New("The calculated MD5 hash of the key did not match the hash that was provided")
-	errSSEKeyMismatch      = errors.New("The client provided key does not match the key provided when the object was encrypted") // this msg is not shown to the client
+	errInsecureSSERequest   = errors.New("SSE-C requests require TLS connections")
+	errEncryptedObject      = errors.New("The object was stored using a form of SSE")
+	errInvalidSSEAlgorithm  = errors.New("The SSE-C algorithm is not valid")
+	errMissingSSEKey        = errors.New("The SSE-C request is missing the customer key")
+	errInvalidSSEKey        = errors.New("The SSE-C key is invalid")
+	errMissingSSEKeyMD5     = errors.New("The SSE-C request is missing the customer key MD5")
+	errSSEKeyMD5Mismatch    = errors.New("The key MD5 does not match the SSE-C key")
+	errSSEKeyMismatch       = errors.New("The SSE-C key is not correct")                  // access denied
+	errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
 
 	// Additional Minio errors for SSE-C requests.
 	errObjectTampered = errors.New("The requested object was modified and may be compromised")
@@ -140,9 +145,20 @@ const (
 	ServerSideEncryptionSealedKey = ReservedMetadataPrefix + "Server-Side-Encryption-Sealed-Key"
 )
 
-// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
-// hash function.
-const SSESealAlgorithmDareSha256 = "DARE-SHA256"
+const (
+	// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
+	// hash function. The key derivation of DARE-SHA256 is not optimal and does not include the object path.
+	// It is considered legacy and should not be used anymore.
+	SSESealAlgorithmDareSha256 = "DARE-SHA256"
+
+	// SSESealAlgorithmDareV2HmacSha256 specifies DAREv2 as authenticated en/decryption scheme and SHA256 as cryptographic
+	// hash function for the HMAC PRF.
+	SSESealAlgorithmDareV2HmacSha256 = "DAREv2-HMAC-SHA256"
+
+	// SSEDomain specifies the domain for the derived key - in this case the
+	// key should be used for SSE-C.
+	SSEDomain = "SSE-C"
+)
 
 // hasSSECustomerHeader returns true if the given HTTP header
 // contains server-side-encryption with customer provided key fields.
@@ -246,13 +262,11 @@ func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
 }
 
 // This function rotates old to new key.
-func rotateKey(oldKey []byte, newKey []byte, metadata map[string]string) error {
-	if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
-		return nil
-	}
+func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map[string]string) error {
 	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
 
-	if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
+	algorithm := metadata[ServerSideEncryptionSealAlgorithm]
+	if algorithm != SSESealAlgorithmDareSha256 && algorithm != SSESealAlgorithmDareV2HmacSha256 {
 		return errObjectTampered
 	}
 	iv, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionIV])
@@ -264,57 +278,68 @@ func rotateKey(oldKey []byte, newKey []byte, metadata map[string]string) error {
 		return errObjectTampered
 	}
 
-	sha := sha256.New() // derive key encryption key
-	sha.Write(oldKey)
-	sha.Write(iv)
-	keyEncryptionKey := sha.Sum(nil)
+	var (
+		minDAREVersion   byte
+		keyEncryptionKey [32]byte
+	)
+	switch algorithm {
+	default:
+		return errObjectTampered
+	case SSESealAlgorithmDareSha256: // legacy key-encryption-key derivation
+		minDAREVersion = sio.Version10
+		sha := sha256.New()
+		sha.Write(oldKey)
+		sha.Write(iv)
+		sha.Sum(keyEncryptionKey[:0])
+	case SSESealAlgorithmDareV2HmacSha256: // key-encryption-key derivation - See: crypto/doc.go
+		minDAREVersion = sio.Version20
+		mac := hmac.New(sha256.New, oldKey)
+		mac.Write(iv)
+		mac.Write([]byte(SSEDomain))
+		mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+		mac.Write([]byte(path.Join(bucket, object)))
+		mac.Sum(keyEncryptionKey[:0])
+	}
 
 	objectEncryptionKey := bytes.NewBuffer(nil) // decrypt object encryption key
 	n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
-		Key: keyEncryptionKey,
+		MinVersion: minDAREVersion,
+		Key:        keyEncryptionKey[:],
 	})
-	if n != 32 || err != nil {
-		// Either the provided key does not match or the object was tampered.
-		// To provide strict AWS S3 compatibility we return: access denied.
-		return errSSEKeyMismatch
+	if n != 32 || err != nil { // Either the provided key does not match or the object was tampered.
+		if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
+			return errInvalidSSEParameters // AWS returns special error for equal but invalid keys.
+		}
+		return errSSEKeyMismatch // To provide strict AWS S3 compatibility we return: access denied.
+	}
+	if subtle.ConstantTimeCompare(oldKey, newKey) == 1 && algorithm != SSESealAlgorithmDareSha256 {
+		return nil // we don't need to rotate keys if newKey == oldKey but we may have to upgrade KDF algorithm
 	}
 
-	nonce := make([]byte, 32) // generate random values for key derivation
-	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
-		return err
-	}
-
-	niv := sha256.Sum256(nonce[:]) // derive key encryption key
-	sha = sha256.New()
-	sha.Write(newKey)
-	sha.Write(niv[:])
-	keyEncryptionKey = sha.Sum(nil)
+	mac := hmac.New(sha256.New, newKey) // key-encryption-key derivation - See: crypto/doc.go
+	mac.Write(iv)
+	mac.Write([]byte(SSEDomain))
+	mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+	mac.Write([]byte(path.Join(bucket, object)))
+	mac.Sum(keyEncryptionKey[:0])
 
 	sealedKeyW := bytes.NewBuffer(nil) // sealedKey := 16 byte header + 32 byte payload + 16 byte tag
 	n, err = sio.Encrypt(sealedKeyW, bytes.NewReader(objectEncryptionKey.Bytes()), sio.Config{
-		Key: keyEncryptionKey,
+		Key: keyEncryptionKey[:],
 	})
 	if n != 64 || err != nil {
 		return errors.New("failed to seal object encryption key") // if this happens there's a bug in the code (may panic ?)
 	}
 
-	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(niv[:])
-	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareSha256
+	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(iv[:])
+	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareV2HmacSha256
 	metadata[ServerSideEncryptionSealedKey] = base64.StdEncoding.EncodeToString(sealedKeyW.Bytes())
 	return nil
 }
 
-func newEncryptMetadata(key []byte, metadata map[string]string) ([]byte, error) {
+func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
 	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
-
-	// security notice:
-	//  - If the first 32 bytes of the random value are ever repeated under the same client-provided
-	//    key the encrypted object will not be tamper-proof. [ P(coll) ~= 1 / 2^(256 / 2)]
-	//  - If the last 32 bytes of the random value are ever repeated under the same client-provided
-	//    key an adversary may be able to extract the object encryption key. This depends on the
-	//    authenticated en/decryption scheme. The DARE format will generate an 8 byte nonce which must
-	//    be repeated in addition to reveal the object encryption key.
-	//    [ P(coll) ~= 1 / 2^((256 + 64) / 2) ]
+	// See crypto/doc.go for detailed description
 	nonce := make([]byte, 32+SSEIVSize) // generate random values for key derivation
 	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
 		return nil, err
@@ -324,11 +349,13 @@ func newEncryptMetadata(key []byte, metadata map[string]string) ([]byte, error)
 	sha.Write(nonce[:32])
 	objectEncryptionKey := sha.Sum(nil)
 
-	iv := sha256.Sum256(nonce[32:]) // derive key encryption key
-	sha = sha256.New()
-	sha.Write(key)
-	sha.Write(iv[:])
-	keyEncryptionKey := sha.Sum(nil)
+	iv := sha256.Sum256(nonce[32:]) // key-encryption-key derivation - See: crypto/doc.go
+	mac := hmac.New(sha256.New, key)
+	mac.Write(iv[:])
+	mac.Write([]byte(SSEDomain))
+	mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+	mac.Write([]byte(path.Join(bucket, object)))
+	keyEncryptionKey := mac.Sum(nil)
 
 	sealedKey := bytes.NewBuffer(nil) // sealedKey := 16 byte header + 32 byte payload + 16 byte tag
 	n, err := sio.Encrypt(sealedKey, bytes.NewReader(objectEncryptionKey), sio.Config{
@@ -339,14 +366,14 @@ func newEncryptMetadata(key []byte, metadata map[string]string) ([]byte, error)
 	}
 
 	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(iv[:])
-	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareSha256
+	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareV2HmacSha256
 	metadata[ServerSideEncryptionSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Bytes())
 
 	return objectEncryptionKey, nil
 }
 
-func newEncryptReader(content io.Reader, key []byte, metadata map[string]string) (io.Reader, error) {
-	objectEncryptionKey, err := newEncryptMetadata(key, metadata)
+func newEncryptReader(content io.Reader, key []byte, bucket, object string, metadata map[string]string) (io.Reader, error) {
+	objectEncryptionKey, err := newEncryptMetadata(key, bucket, object, metadata)
 	if err != nil {
 		return nil, err
 	}
@@ -362,29 +389,26 @@ func newEncryptReader(content io.Reader, key []byte, metadata map[string]string)
 // EncryptRequest takes the client provided content and encrypts the data
 // with the client provided key. It also marks the object as client-side-encrypted
 // and sets the correct headers.
-func EncryptRequest(content io.Reader, r *http.Request, metadata map[string]string) (io.Reader, error) {
+func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (io.Reader, error) {
 	key, err := ParseSSECustomerRequest(r)
 	if err != nil {
 		return nil, err
 	}
-	return newEncryptReader(content, key, metadata)
+	return newEncryptReader(content, key, bucket, object, metadata)
 }
 
 // DecryptCopyRequest decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
-func DecryptCopyRequest(client io.Writer, r *http.Request, metadata map[string]string) (io.WriteCloser, error) {
+func DecryptCopyRequest(client io.Writer, r *http.Request, bucket, object string, metadata map[string]string) (io.WriteCloser, error) {
 	key, err := ParseSSECopyCustomerRequest(r)
 	if err != nil {
 		return nil, err
 	}
 	delete(metadata, SSECopyCustomerKey) // make sure we do not save the key by accident
-	return newDecryptWriter(client, key, 0, metadata)
+	return newDecryptWriter(client, key, bucket, object, 0, metadata)
 }
 
-func decryptObjectInfo(key []byte, metadata map[string]string) ([]byte, error) {
-	if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
-		return nil, errObjectTampered
-	}
+func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
 	iv, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionIV])
 	if err != nil || len(iv) != SSEIVSize {
 		return nil, errObjectTampered
@@ -394,14 +418,33 @@ func decryptObjectInfo(key []byte, metadata map[string]string) ([]byte, error) {
 		return nil, errObjectTampered
 	}
 
-	sha := sha256.New() // derive key encryption key
-	sha.Write(key)
-	sha.Write(iv)
-	keyEncryptionKey := sha.Sum(nil)
+	var (
+		minDAREVersion   byte
+		keyEncryptionKey [32]byte
+	)
+	switch algorithm := metadata[ServerSideEncryptionSealAlgorithm]; algorithm {
+	default:
+		return nil, errObjectTampered
+	case SSESealAlgorithmDareSha256: // legacy key-encryption-key derivation
+		minDAREVersion = sio.Version10
+		sha := sha256.New()
+		sha.Write(key)
+		sha.Write(iv)
+		sha.Sum(keyEncryptionKey[:0])
+	case SSESealAlgorithmDareV2HmacSha256: // key-encryption-key derivation - See: crypto/doc.go
+		minDAREVersion = sio.Version20
+		mac := hmac.New(sha256.New, key)
+		mac.Write(iv)
+		mac.Write([]byte(SSEDomain))
+		mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+		mac.Write([]byte(path.Join(bucket, object)))
+		mac.Sum(keyEncryptionKey[:0])
+	}
 
 	objectEncryptionKey := bytes.NewBuffer(nil) // decrypt object encryption key
 	n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
-		Key: keyEncryptionKey,
+		MinVersion: minDAREVersion,
+		Key:        keyEncryptionKey[:],
 	})
 	if n != 32 || err != nil {
 		// Either the provided key does not match or the object was tampered.
@@ -411,11 +454,10 @@ func decryptObjectInfo(key []byte, metadata map[string]string) ([]byte, error) {
 	return objectEncryptionKey.Bytes(), nil
 }
 
-func newDecryptWriter(client io.Writer, key []byte, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
-	objectEncryptionKey, err := decryptObjectInfo(key, metadata)
+func newDecryptWriter(client io.Writer, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
+	objectEncryptionKey, err := decryptObjectInfo(key, bucket, object, metadata)
 	if err != nil {
 		return nil, err
-
 	}
 	return newDecryptWriterWithObjectKey(client, objectEncryptionKey, seqNumber, metadata)
 }
@@ -438,19 +480,19 @@ func newDecryptWriterWithObjectKey(client io.Writer, objectEncryptionKey []byte,
 
 // DecryptRequestWithSequenceNumber decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
-func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
+func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, bucket, object string, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
 	key, err := ParseSSECustomerRequest(r)
 	if err != nil {
 		return nil, err
 	}
 	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
-	return newDecryptWriter(client, key, seqNumber, metadata)
+	return newDecryptWriter(client, key, bucket, object, seqNumber, metadata)
 }
 
 // DecryptRequest decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
-func DecryptRequest(client io.Writer, r *http.Request, metadata map[string]string) (io.WriteCloser, error) {
-	return DecryptRequestWithSequenceNumber(client, r, 0, metadata)
+func DecryptRequest(client io.Writer, r *http.Request, bucket, object string, metadata map[string]string) (io.WriteCloser, error) {
+	return DecryptRequestWithSequenceNumber(client, r, bucket, object, 0, metadata)
 }
 
 // DecryptBlocksWriter - decrypts multipart parts, while implementing a io.Writer compatible interface.
@@ -464,9 +506,10 @@ type DecryptBlocksWriter struct {
 	// Current part index
 	partIndex int
 	// Parts information
-	parts    []objectPartInfo
-	req      *http.Request
-	metadata map[string]string
+	parts          []objectPartInfo
+	req            *http.Request
+	bucket, object string
+	metadata       map[string]string
 
 	partEncRelOffset int64
 
@@ -494,7 +537,7 @@ func (w *DecryptBlocksWriter) buildDecrypter(partID int) error {
 		return err
 	}
 
-	objectEncryptionKey, err := decryptObjectInfo(key, m)
+	objectEncryptionKey, err := decryptObjectInfo(key, w.bucket, w.object, m)
 	if err != nil {
 		return err
 	}
@@ -589,29 +632,26 @@ func (w *DecryptBlocksWriter) Close() error {
 // DecryptAllBlocksCopyRequest - setup a struct which can decrypt many concatenated encrypted data
 // parts information helps to know the boundaries of each encrypted data block, this function decrypts
 // all parts starting from part-1.
-func DecryptAllBlocksCopyRequest(client io.Writer, r *http.Request, objInfo ObjectInfo) (io.WriteCloser, int64, error) {
-	w, _, size, err := DecryptBlocksRequest(client, r, 0, objInfo.Size, objInfo, true)
+func DecryptAllBlocksCopyRequest(client io.Writer, r *http.Request, bucket, object string, objInfo ObjectInfo) (io.WriteCloser, int64, error) {
+	w, _, size, err := DecryptBlocksRequest(client, r, bucket, object, 0, objInfo.Size, objInfo, true)
 	return w, size, err
 }
 
 // DecryptBlocksRequest - setup a struct which can decrypt many concatenated encrypted data
 // parts information helps to know the boundaries of each encrypted data block.
-func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length int64, objInfo ObjectInfo, copySource bool) (io.WriteCloser, int64, int64, error) {
-	seqNumber, encStartOffset, encLength := getEncryptedStartOffset(startOffset, length)
-
-	// Encryption length cannot be bigger than the file size, if it is
-	// which is allowed in AWS S3, we simply default to EncryptedSize().
-	if encLength+encStartOffset > objInfo.EncryptedSize() {
-		encLength = objInfo.EncryptedSize() - encStartOffset
-	}
+func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object string, startOffset, length int64, objInfo ObjectInfo, copySource bool) (io.WriteCloser, int64, int64, error) {
+	var seqNumber uint32
+	var encStartOffset, encLength int64
 
 	if len(objInfo.Parts) == 0 || !objInfo.IsEncryptedMultipart() {
+		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
+
 		var writer io.WriteCloser
 		var err error
 		if copySource {
-			writer, err = DecryptCopyRequest(client, r, objInfo.UserDefined)
+			writer, err = DecryptCopyRequest(client, r, bucket, object, objInfo.UserDefined)
 		} else {
-			writer, err = DecryptRequestWithSequenceNumber(client, r, seqNumber, objInfo.UserDefined)
+			writer, err = DecryptRequestWithSequenceNumber(client, r, bucket, object, seqNumber, objInfo.UserDefined)
 		}
 		if err != nil {
 			return nil, 0, 0, err
@@ -619,6 +659,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length
 		return writer, encStartOffset, encLength, nil
 	}
 
+	seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
 	var partStartIndex int
 	var partStartOffset = startOffset
 	// Skip parts until final offset maps to a particular part offset.
@@ -651,6 +692,8 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length
 		parts:             objInfo.Parts,
 		partIndex:         partStartIndex,
 		req:               r,
+		bucket:            bucket,
+		object:            object,
 		customerKeyHeader: r.Header.Get(SSECustomerKey),
 		copySource:        copySource,
 	}
@@ -671,15 +714,62 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length
 		w.customerKeyHeader = r.Header.Get(SSECopyCustomerKey)
 	}
 
-	if err := w.buildDecrypter(partStartIndex + 1); err != nil {
+	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
 		return nil, 0, 0, err
 	}
 
 	return w, encStartOffset, encLength, nil
 }
 
-// getEncryptedStartOffset - fetch sequence number, encrypted start offset and encrypted length.
-func getEncryptedStartOffset(offset, length int64) (seqNumber uint32, encOffset int64, encLength int64) {
+// getEncryptedMultipartsOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
+func getEncryptedMultipartsOffsetLength(offset, length int64, obj ObjectInfo) (uint32, int64, int64) {
+
+	// Calculate encrypted offset of a multipart object
+	computeEncOffset := func(off int64, obj ObjectInfo) (seqNumber uint32, encryptedOffset int64, err error) {
+		var curPartEndOffset uint64
+		var prevPartsEncSize int64
+		for _, p := range obj.Parts {
+			size, decErr := sio.DecryptedSize(uint64(p.Size))
+			if decErr != nil {
+				err = errObjectTampered // assign correct error type
+				return
+			}
+			if off < int64(curPartEndOffset+size) {
+				seqNumber, encryptedOffset, _ = getEncryptedSinglePartOffsetLength(off-int64(curPartEndOffset), 1, obj)
+				encryptedOffset += int64(prevPartsEncSize)
+				break
+			}
+			curPartEndOffset += size
+			prevPartsEncSize += p.Size
+		}
+		return
+	}
+
+	// Calculate the encrypted start offset corresponding to the plain offset
+	seqNumber, encStartOffset, _ := computeEncOffset(offset, obj)
+	// Calculate also the encrypted end offset corresponding to plain offset + plain length
+	_, encEndOffset, _ := computeEncOffset(offset+length-1, obj)
+
+	// encLength is the diff between encrypted end offset and encrypted start offset + one package size
+	// to ensure all encrypted data are covered
+	encLength := encEndOffset - encStartOffset + (64*1024 + 32)
+
+	// Calculate total size of all parts
+	var totalPartsLength int64
+	for _, p := range obj.Parts {
+		totalPartsLength += p.Size
+	}
+
+	// Set encLength to maximum possible value if it exceeded total parts size
+	if encLength+encStartOffset > totalPartsLength {
+		encLength = totalPartsLength - encStartOffset
+	}
+
+	return seqNumber, encStartOffset, encLength
+}
+
+// getEncryptedSinglePartOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
+func getEncryptedSinglePartOffsetLength(offset, length int64, objInfo ObjectInfo) (seqNumber uint32, encOffset int64, encLength int64) {
 	onePkgSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
 
 	seqNumber = uint32(offset / sseDAREPackageBlockSize)
@@ -697,6 +787,9 @@ func getEncryptedStartOffset(offset, length int64) (seqNumber uint32, encOffset
 		encLength += onePkgSize
 	}
 
+	if encLength+encOffset > objInfo.EncryptedSize() {
+		encLength = objInfo.EncryptedSize() - encOffset
+	}
 	return seqNumber, encOffset, encLength
 }
 
@@ -737,16 +830,27 @@ func (li *ListPartsInfo) IsEncrypted() bool {
 // DecryptedSize returns the size of the object after decryption in bytes.
 // It returns an error if the object is not encrypted or marked as encrypted
 // but has an invalid size.
-// DecryptedSize panics if the referred object is not encrypted.
 func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if !o.IsEncrypted() {
-		panic("cannot compute decrypted size of an object which is not encrypted")
+		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
-	size, err := sio.DecryptedSize(uint64(o.Size))
-	if err != nil {
-		err = errObjectTampered // assign correct error type
+	if len(o.Parts) == 0 || !o.IsEncryptedMultipart() {
+		size, err := sio.DecryptedSize(uint64(o.Size))
+		if err != nil {
+			err = errObjectTampered // assign correct error type
+		}
+		return int64(size), err
 	}
-	return int64(size), err
+
+	var size int64
+	for _, part := range o.Parts {
+		partSize, err := sio.DecryptedSize(uint64(part.Size))
+		if err != nil {
+			return 0, errObjectTampered
+		}
+		size += int64(partSize)
+	}
+	return size, nil
 }
 
 // EncryptedSize returns the size of the object after encryption.
@@ -755,7 +859,11 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 func (o *ObjectInfo) EncryptedSize() int64 {
 	size, err := sio.EncryptedSize(uint64(o.Size))
 	if err != nil {
-		panic(err) // Since AWS S3 allows parts to be 5GB at most this cannot happen - sio max. size is 256 TB
+		// This cannot happen since AWS S3 allows parts to be 5GB at most
+		// sio max. size is 256 TB
+		reqInfo := (&logger.ReqInfo{}).AppendTags("size", strconv.FormatUint(size, 10))
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.CriticalIf(ctx, err)
 	}
 	return int64(size)
 }

cmd/encryption-v1_test.go

@@ -308,7 +308,7 @@ func TestEncryptRequest(t *testing.T) {
 		for k, v := range test.header {
 			req.Header.Set(k, v)
 		}
-		_, err := EncryptRequest(content, req, test.metadata)
+		_, err := EncryptRequest(content, req, "bucket", "object", test.metadata)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
@@ -328,11 +328,14 @@ func TestEncryptRequest(t *testing.T) {
 }
 
 var decryptRequestTests = []struct {
-	header     map[string]string
-	metadata   map[string]string
-	shouldFail bool
+	bucket, object string
+	header         map[string]string
+	metadata       map[string]string
+	shouldFail     bool
 }{
 	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
@@ -346,6 +349,23 @@ var decryptRequestTests = []struct {
 		shouldFail: false,
 	},
 	{
+		bucket: "bucket",
+		object: "object",
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
+			SSECustomerKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareV2HmacSha256,
+			ServerSideEncryptionIV:            "qEqmsONcorqlcZXJxaw32H04eyXyXwUgjHzlhkaIYrU=",
+			ServerSideEncryptionSealedKey:     "IAAfAIM14ugTGcM/dIrn4iQMrkl1sjKyeBQ8FBEvRebYj8vWvxG+0cJRpC6NXRU1wJN50JaUOATjO7kz0wZ2mA==",
+		},
+		shouldFail: false,
+	},
+	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
@@ -359,6 +379,8 @@ var decryptRequestTests = []struct {
 		shouldFail: true,
 	},
 	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
@@ -372,6 +394,8 @@ var decryptRequestTests = []struct {
 		shouldFail: true,
 	},
 	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
@@ -384,21 +408,39 @@ var decryptRequestTests = []struct {
 		},
 		shouldFail: true,
 	},
+	{
+		bucket: "bucket",
+		object: "object-2",
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
+			SSECustomerKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareV2HmacSha256,
+			ServerSideEncryptionIV:            "qEqmsONcorqlcZXJxaw32H04eyXyXwUgjHzlhkaIYrU=",
+			ServerSideEncryptionSealedKey:     "IAAfAIM14ugTGcM/dIrn4iQMrkl1sjKyeBQ8FBEvRebYj8vWvxG+0cJRpC6NXRU1wJN50JaUOATjO7kz0wZ2mA==",
+		},
+		shouldFail: true,
+	},
 }
 
 func TestDecryptRequest(t *testing.T) {
 	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
 	globalIsSSL = true
-	for i, test := range decryptRequestTests {
+	for i, test := range decryptRequestTests[1:] {
 		client := bytes.NewBuffer(nil)
 		req := &http.Request{Header: http.Header{}}
 		for k, v := range test.header {
 			req.Header.Set(k, v)
 		}
-		_, err := DecryptRequest(client, req, test.metadata)
+		_, err := DecryptRequest(client, req, test.bucket, test.object, test.metadata)
 		if err != nil && !test.shouldFail {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
+		if err == nil && test.shouldFail {
+			t.Fatalf("Test %d: should fail but passed", i)
+		}
 		if key, ok := test.metadata[SSECustomerKey]; ok {
 			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
 		}

cmd/endpoint-ellipses.go

@@ -70,11 +70,10 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 	}
 
 	setIndexes = make([][]uint64, len(totalSizes))
-	for i, totalSize := range totalSizes {
+	for _, totalSize := range totalSizes {
 		// Check if totalSize has minimum range upto setSize
 		if totalSize < setSizes[0] {
-			return nil, fmt.Errorf("Invalid inputs (%s). Ellipses range or number of args %d should be atleast divisible by least possible set size %d",
-				args[i], totalSize, setSizes[0])
+			return nil, uiErrInvalidNumberOfErasureEndpoints(nil)
 		}
 	}
 
@@ -103,8 +102,7 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		return nil, fmt.Errorf("Invalid inputs (%s). Ellipses range or number of args %d should be atleast divisible by least possible set size %d",
-			args, setSize, setSizes[0])
+		return nil, uiErrInvalidNumberOfErasureEndpoints(nil)
 	}
 
 	for i := range totalSizes {
@@ -167,14 +165,14 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) {
 	for i, arg := range args {
 		patterns, perr := ellipses.FindEllipsesPatterns(arg)
 		if perr != nil {
-			return endpointSet{}, perr
+			return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(perr.Error())
 		}
 		argPatterns[i] = patterns
 	}
 
 	ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns))
 	if err != nil {
-		return endpointSet{}, err
+		return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}
 
 	ep.argPatterns = argPatterns
@@ -223,7 +221,7 @@ func getAllSets(args ...string) ([][]string, error) {
 	for _, sargs := range setArgs {
 		for _, arg := range sargs {
 			if uniqueArgs.Contains(arg) {
-				return nil, fmt.Errorf("Input args (%s) has duplicate ellipses", args)
+				return nil, uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args))
 			}
 			uniqueArgs.Add(arg)
 		}

cmd/endpoint.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"os"
 	"path"
 	"path/filepath"
 	"runtime"
@@ -27,6 +28,7 @@ import (
 	"strings"
 
 	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/mountinfo"
 )
 
@@ -251,14 +253,14 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			return serverAddr, endpoints, setupType, err
 		}
 		if endpoint.Type() != PathEndpointType {
-			return serverAddr, endpoints, setupType, fmt.Errorf("use path style endpoint for FS setup")
+			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
 		}
 		endpoints = append(endpoints, endpoint)
 		setupType = FSSetupType
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(endpoints); err != nil {
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg(err.Error())
 		}
 		return serverAddr, endpoints, setupType, nil
 	}
@@ -269,12 +271,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		var eps EndpointList
 		eps, err = NewEndpointList(iargs...)
 		if err != nil {
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(eps); err != nil {
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
 		for _, ep := range eps {
@@ -315,7 +317,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 
 	// No local endpoint found.
 	if localEndpointCount == 0 {
-		return serverAddr, endpoints, setupType, fmt.Errorf("no endpoint found for this host")
+		return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg("no endpoint pointing to the local machine is found")
 	}
 
 	// Check whether same path is not used in endpoints of a host on different port.
@@ -330,8 +332,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			hostIPSet, _ := getHostIP4(host)
 			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
 				if !IPSet.Intersection(hostIPSet).IsEmpty() {
-					err = fmt.Errorf("path '%s' can not be served by different port on same address", endpoint.Path)
-					return serverAddr, endpoints, setupType, err
+					return serverAddr, endpoints, setupType,
+						uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
 				}
 				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
 			} else {
@@ -348,8 +350,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 				continue
 			}
 			if localPathSet.Contains(endpoint.Path) {
-				err = fmt.Errorf("path '%s' cannot be served by different address on same server", endpoint.Path)
-				return serverAddr, endpoints, setupType, err
+				return serverAddr, endpoints, setupType,
+					uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
 			}
 			localPathSet.Add(endpoint.Path)
 		}
@@ -359,12 +361,11 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 	{
 		if !localPortSet.Contains(serverAddrPort) {
 			if len(localPortSet) > 1 {
-				err = fmt.Errorf("port number in server address must match with one of the port in local endpoints")
-			} else {
-				err = fmt.Errorf("server address and local endpoint have different ports")
+				return serverAddr, endpoints, setupType,
+					uiErrInvalidErasureEndpoints(nil).Msg("port number in server address must match with one of the port in local endpoints")
 			}
-
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType,
+				uiErrInvalidErasureEndpoints(nil).Msg("server address and local endpoint have different ports")
 		}
 	}
 
@@ -373,7 +374,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		// If all endpoints have same port number, then this is XL setup using URL style endpoints.
 		if len(localPortSet) == 1 {
 			if len(localServerAddrSet) > 1 {
-				// TODO: Eventhough all endpoints are local, the local host is referred by different IP/name.
+				// TODO: Even though all endpoints are local, the local host is referred by different IP/name.
 				// eg '172.0.0.1', 'localhost' and 'mylocalhostname' point to same local host.
 				//
 				// In this case, we bind to 0.0.0.0 ie to all interfaces.
@@ -387,7 +388,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			return serverAddr, endpoints, setupType, nil
 		}
 
-		// Eventhough all endpoints are local, but those endpoints use different ports.
+		// Even though all endpoints are local, but those endpoints use different ports.
 		// This means it is DistXL setup.
 	} else {
 		// This is DistXL setup.
@@ -399,7 +400,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			}
 
 			ipList, err := getHostIP4(host)
-			fatalIf(err, "unexpected error when resolving host '%s'", host)
+			logger.FatalIf(err, "unexpected error when resolving host '%s'", host)
 
 			// Filter ipList by IPs those start with '127.'.
 			loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool {
@@ -442,6 +443,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		return serverAddr, endpoints, setupType, err
 	}
 
+	updateDomainIPs(uniqueArgs)
+
 	setupType = DistXLSetupType
 	return serverAddr, endpoints, setupType, nil
 }
@@ -493,3 +496,22 @@ func GetRemotePeers(endpoints EndpointList) []string {
 
 	return peerSet.ToSlice()
 }
+
+// In federated and distributed setup, update IP addresses of the hosts passed in command line
+// if MINIO_PUBLIC_IPS are not set manually
+func updateDomainIPs(endPoints set.StringSet) {
+	_, dok := os.LookupEnv("MINIO_DOMAIN")
+	_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if dok && eok && !iok {
+		globalDomainIPs = set.NewStringSet()
+		for e := range endPoints {
+			host, _, _ := net.SplitHostPort(e)
+			ipList, _ := getHostIP4(host)
+			remoteIPList := ipList.FuncMatch(func(ip string, matchString string) bool {
+				return !strings.HasPrefix(ip, "127.")
+			}, "")
+			globalDomainIPs.Add(remoteIPList.ToSlice()[0])
+		}
+	}
+}

cmd/endpoint_test.go

@@ -225,8 +225,8 @@ func TestCreateEndpoints(t *testing.T) {
 		{"localhost:10000", [][]string{{"./d1"}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
 		{"localhost:10000", [][]string{{`\d1`}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `\d1`}, IsLocal: true}}, FSSetupType, nil},
 		{"localhost:10000", [][]string{{`.\d1`}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `.\d1`}, IsLocal: true}}, FSSetupType, nil},
-		{":8080", [][]string{{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
-		{":8080", [][]string{{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
+		{":8080", [][]string{{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint pointing to the local machine is found")},
+		{":8080", [][]string{{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint pointing to the local machine is found")},
 		{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
 		{"localhost:9000", [][]string{{"https://127.0.0.1:8000/d1", "https://localhost:9001/d2", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("port number in server address must match with one of the port in local endpoints")},
 		{"localhost:10000", [][]string{{"https://127.0.0.1:8000/d1", "https://localhost:8000/d2", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("server address and local endpoint have different ports")},

cmd/erasure-createfile.go

@@ -17,18 +17,20 @@
 package cmd
 
 import (
+	"context"
 	"hash"
 	"io"
 
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // CreateFile creates a new bitrot encoded file spread over all available disks. CreateFile will create
 // the file at the given volume and path. It will read from src until an io.EOF occurs. The given algorithm will
 // be used to protect the erasure encoded file.
-func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) {
+func (s *ErasureStorage) CreateFile(ctx context.Context, src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) {
 	if !algorithm.Available() {
-		return f, errors.Trace(errBitrotHashAlgoInvalid)
+		logger.LogIf(ctx, errBitrotHashAlgoInvalid)
+		return f, errBitrotHashAlgoInvalid
 	}
 	f.Checksums = make([][]byte, len(s.disks))
 	hashers := make([]hash.Hash, len(s.disks))
@@ -50,21 +52,22 @@ func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer [
 			}
 			blocks = make([][]byte, len(s.disks)) // write empty block
 		} else if err == nil || (n > 0 && err == io.ErrUnexpectedEOF) {
-			blocks, err = s.ErasureEncode(buffer[:n])
+			blocks, err = s.ErasureEncode(ctx, buffer[:n])
 			if err != nil {
 				return f, err
 			}
 		} else {
-			return f, errors.Trace(err)
+			logger.LogIf(ctx, err)
+			return f, err
 		}
 
 		for i := range errChans { // span workers
-			go erasureAppendFile(s.disks[i], volume, path, hashers[i], blocks[i], errChans[i])
+			go erasureAppendFile(ctx, s.disks[i], volume, path, hashers[i], blocks[i], errChans[i])
 		}
 		for i := range errChans { // wait until all workers are finished
 			errs[i] = <-errChans[i]
 		}
-		if err = reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum); err != nil {
+		if err = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum); err != nil {
 			return f, err
 		}
 		s.disks = evalDisks(s.disks, errs)
@@ -83,9 +86,10 @@ func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer [
 
 // erasureAppendFile appends the content of buf to the file on the given disk and updates computes
 // the hash of the written data. It sends the write error (or nil) over the error channel.
-func erasureAppendFile(disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) {
+func erasureAppendFile(ctx context.Context, disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) {
 	if disk == OfflineDisk {
-		errChan <- errors.Trace(errDiskNotFound)
+		logger.LogIf(ctx, errDiskNotFound)
+		errChan <- errDiskNotFound
 		return
 	}
 	err := disk.AppendFile(volume, path, buf)

cmd/erasure-createfile_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"io"
 	"testing"
@@ -70,7 +71,7 @@ func TestErasureCreateFile(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		storage, err := NewErasureStorage(context.Background(), setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -82,7 +83,7 @@ func TestErasureCreateFile(t *testing.T) {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
 		}
-		file, err := storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object", buffer, test.algorithm, test.dataBlocks+1)
+		file, err := storage.CreateFile(context.Background(), bytes.NewReader(data[test.offset:]), "testbucket", "object", buffer, test.algorithm, test.dataBlocks+1)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
 		}
@@ -100,7 +101,7 @@ func TestErasureCreateFile(t *testing.T) {
 			if test.offDisks > 0 {
 				storage.disks[0] = OfflineDisk
 			}
-			file, err = storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object2", buffer, test.algorithm, test.dataBlocks+1)
+			file, err = storage.CreateFile(context.Background(), bytes.NewReader(data[test.offset:]), "testbucket", "object2", buffer, test.algorithm, test.dataBlocks+1)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
 			}
@@ -125,7 +126,7 @@ func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b
 		b.Fatalf("failed to create test setup: %v", err)
 	}
 	defer setup.Remove()
-	storage, err := NewErasureStorage(setup.disks, data, parity, blockSizeV1)
+	storage, err := NewErasureStorage(context.Background(), setup.disks, data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -143,7 +144,7 @@ func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b
 	b.SetBytes(size)
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		_, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
+		_, err := storage.CreateFile(context.Background(), bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
 		if err != nil {
 			panic(err)
 		}

cmd/erasure-healfile.go

@@ -17,11 +17,12 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"hash"
 	"strings"
 
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // HealFile tries to reconstruct an erasure-coded file spread over all
@@ -45,12 +46,13 @@ import (
 //
 // It returns bitrot checksums for the non-nil staleDisks on which
 // healing succeeded.
-func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, blocksize int64,
+func (s ErasureStorage) HealFile(ctx context.Context, staleDisks []StorageAPI, volume, path string, blocksize int64,
 	dstVol, dstPath string, size int64, alg BitrotAlgorithm, checksums [][]byte) (
 	f ErasureFileInfo, err error) {
 
 	if !alg.Available() {
-		return f, errors.Trace(errBitrotHashAlgoInvalid)
+		logger.LogIf(ctx, errBitrotHashAlgoInvalid)
+		return f, errBitrotHashAlgoInvalid
 	}
 
 	// Initialization
@@ -84,7 +86,7 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
 	}
 	readLen += lastChunkSize
 	var buffers [][]byte
-	buffers, _, err = s.readConcurrent(volume, path, 0, readLen, verifiers)
+	buffers, _, err = s.readConcurrent(ctx, volume, path, 0, readLen, verifiers)
 	if err != nil {
 		return f, err
 	}
@@ -131,7 +133,7 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
 		}
 		buffOffset += csize
 
-		if err = s.ErasureDecodeDataAndParityBlocks(blocks); err != nil {
+		if err = s.ErasureDecodeDataAndParityBlocks(ctx, blocks); err != nil {
 			return f, err
 		}
 
@@ -155,7 +157,9 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
 		// If all disks had write errors we quit.
 		if !writeSucceeded {
 			// build error from all write errors
-			return f, errors.Trace(joinWriteErrors(writeErrors))
+			err := joinWriteErrors(writeErrors)
+			logger.LogIf(ctx, err)
+			return f, err
 		}
 	}
 

cmd/erasure-healfile_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"io"
 	"reflect"
@@ -74,7 +75,7 @@ func TestErasureHealFile(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
 		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
+		storage, err := NewErasureStorage(context.Background(), setup.disks, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -89,7 +90,7 @@ func TestErasureHealFile(t *testing.T) {
 			algorithm = DefaultBitrotAlgorithm
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
-		file, err := storage.CreateFile(bytes.NewReader(data), "testbucket", "testobject", buffer, algorithm, test.dataBlocks+1)
+		file, err := storage.CreateFile(context.Background(), bytes.NewReader(data), "testbucket", "testobject", buffer, algorithm, test.dataBlocks+1)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
@@ -113,7 +114,7 @@ func TestErasureHealFile(t *testing.T) {
 		}
 
 		// test case setup is complete - now call Healfile()
-		info, err := storage.HealFile(staleDisks, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums)
+		info, err := storage.HealFile(context.Background(), staleDisks, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but it failed with: %v", i, err)
 		}

cmd/erasure-readfile.go

@@ -17,9 +17,10 @@
 package cmd
 
 import (
+	"context"
 	"io"
 
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 type errIdx struct {
@@ -27,7 +28,7 @@ type errIdx struct {
 	err error
 }
 
-func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64,
+func (s ErasureStorage) readConcurrent(ctx context.Context, volume, path string, offset, length int64,
 	verifiers []*BitrotVerifier) (buffers [][]byte, needsReconstruction bool,
 	err error) {
 
@@ -39,7 +40,8 @@ func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64
 		stageBuffers[i] = make([]byte, length)
 		disk := s.disks[i]
 		if disk == OfflineDisk {
-			errChan <- errIdx{i, errors.Trace(errDiskNotFound)}
+			logger.LogIf(ctx, errDiskNotFound)
+			errChan <- errIdx{i, errDiskNotFound}
 			return
 		}
 		_, rerr := disk.ReadFile(volume, path, offset, stageBuffers[i], verifiers[i])
@@ -75,7 +77,8 @@ func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64
 	}
 	if successCount != s.dataBlocks {
 		// Not enough disks returns data.
-		err = errors.Trace(errXLReadQuorum)
+		err = errXLReadQuorum
+		logger.LogIf(ctx, err)
 	}
 	return
 }
@@ -86,18 +89,21 @@ func (s ErasureStorage) readConcurrent(volume, path string, offset, length int64
 // integrity of the given file. ReadFile will read data from the given
 // offset up to the given length. If parts of the file are corrupted
 // ReadFile tries to reconstruct the data.
-func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset,
+func (s ErasureStorage) ReadFile(ctx context.Context, writer io.Writer, volume, path string, offset,
 	length, totalLength int64, checksums [][]byte, algorithm BitrotAlgorithm,
 	blocksize int64) (f ErasureFileInfo, err error) {
 
 	if offset < 0 || length < 0 {
-		return f, errors.Trace(errUnexpected)
+		logger.LogIf(ctx, errUnexpected)
+		return f, errUnexpected
 	}
 	if offset+length > totalLength {
-		return f, errors.Trace(errUnexpected)
+		logger.LogIf(ctx, errUnexpected)
+		return f, errUnexpected
 	}
 	if !algorithm.Available() {
-		return f, errors.Trace(errBitrotHashAlgoInvalid)
+		logger.LogIf(ctx, errBitrotHashAlgoInvalid)
+		return f, errBitrotHashAlgoInvalid
 	}
 
 	f.Checksums = make([][]byte, len(s.disks))
@@ -145,7 +151,7 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset,
 
 	var buffers [][]byte
 	var needsReconstruction bool
-	buffers, needsReconstruction, err = s.readConcurrent(volume, path,
+	buffers, needsReconstruction, err = s.readConcurrent(ctx, volume, path,
 		partDataStartIndex, partDataLength, verifiers)
 	if err != nil {
 		// Could not read enough disks.
@@ -194,7 +200,8 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset,
 
 		if needsReconstruction {
 			if err = s.ErasureDecodeDataBlocks(blocks); err != nil {
-				return f, errors.Trace(err)
+				logger.LogIf(ctx, err)
+				return f, err
 			}
 		}
 
@@ -210,7 +217,7 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset,
 				writeLength = lastBlockLength - writeStart
 			}
 		}
-		n, err := writeDataBlocks(writer, blocks, s.dataBlocks, writeStart, writeLength)
+		n, err := writeDataBlocks(ctx, writer, blocks, s.dataBlocks, writeStart, writeLength)
 		if err != nil {
 			return f, err
 		}

cmd/erasure-readfile_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	crand "crypto/rand"
 	"io"
 	"math/rand"
@@ -86,7 +87,7 @@ func TestErasureReadFile(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		storage, err := NewErasureStorage(context.Background(), setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -102,13 +103,13 @@ func TestErasureReadFile(t *testing.T) {
 			writeAlgorithm = DefaultBitrotAlgorithm
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
-		file, err := storage.CreateFile(bytes.NewReader(data[:]), "testbucket", "object", buffer, writeAlgorithm, test.dataBlocks+1)
+		file, err := storage.CreateFile(context.Background(), bytes.NewReader(data[:]), "testbucket", "object", buffer, writeAlgorithm, test.dataBlocks+1)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file: %v", i, err)
 		}
 		writer := bytes.NewBuffer(nil)
-		readInfo, err := storage.ReadFile(writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
+		readInfo, err := storage.ReadFile(context.Background(), writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
 		}
@@ -134,7 +135,7 @@ func TestErasureReadFile(t *testing.T) {
 			if test.offDisks > 0 {
 				storage.disks[0] = OfflineDisk
 			}
-			readInfo, err = storage.ReadFile(writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
+			readInfo, err = storage.ReadFile(context.Background(), writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
 			}
@@ -174,7 +175,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	}
 	defer setup.Remove()
 
-	storage, err := NewErasureStorage(setup.disks, dataBlocks, parityBlocks, blockSize)
+	storage, err := NewErasureStorage(context.Background(), setup.disks, dataBlocks, parityBlocks, blockSize)
 	if err != nil {
 		t.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -191,7 +192,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 	// Create a test file to read from.
 	buffer := make([]byte, blockSize, 2*blockSize)
-	file, err := storage.CreateFile(bytes.NewReader(data), "testbucket", "testobject", buffer, DefaultBitrotAlgorithm, dataBlocks+1)
+	file, err := storage.CreateFile(context.Background(), bytes.NewReader(data), "testbucket", "testobject", buffer, DefaultBitrotAlgorithm, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -211,7 +212,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 		expected := data[offset : offset+readLen]
 
-		_, err = storage.ReadFile(buf, "testbucket", "testobject", offset, readLen, length, file.Checksums, DefaultBitrotAlgorithm, blockSize)
+		_, err = storage.ReadFile(context.Background(), buf, "testbucket", "testobject", offset, readLen, length, file.Checksums, DefaultBitrotAlgorithm, blockSize)
 		if err != nil {
 			t.Fatal(err, offset, readLen)
 		}
@@ -231,14 +232,14 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b
 		b.Fatalf("failed to create test setup: %v", err)
 	}
 	defer setup.Remove()
-	storage, err := NewErasureStorage(setup.disks, data, parity, blockSizeV1)
+	storage, err := NewErasureStorage(context.Background(), setup.disks, data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}
 
 	content := make([]byte, size)
 	buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
-	file, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
+	file, err := storage.CreateFile(context.Background(), bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
 	if err != nil {
 		b.Fatalf("failed to create erasure test file: %v", err)
 	}
@@ -255,7 +256,7 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b
 	b.SetBytes(size)
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		if file, err = storage.ReadFile(bytes.NewBuffer(content[:0]), "testbucket", "object", 0, size, size, checksums, DefaultBitrotAlgorithm, blockSizeV1); err != nil {
+		if file, err = storage.ReadFile(context.Background(), bytes.NewBuffer(content[:0]), "testbucket", "object", 0, size, size, checksums, DefaultBitrotAlgorithm, blockSizeV1); err != nil {
 			panic(err)
 		}
 	}

cmd/erasure-utils.go

@@ -18,10 +18,11 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"io"
 
 	"github.com/klauspost/reedsolomon"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // getDataBlockLen - get length of data blocks from encoded blocks.
@@ -36,20 +37,23 @@ func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
 
 // Writes all the data blocks from encoded blocks until requested
 // outSize length. Provides a way to skip bytes until the offset.
-func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
+func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
 	// Offset and out size cannot be negative.
 	if offset < 0 || length < 0 {
-		return 0, errors.Trace(errUnexpected)
+		logger.LogIf(ctx, errUnexpected)
+		return 0, errUnexpected
 	}
 
 	// Do we have enough blocks?
 	if len(enBlocks) < dataBlocks {
-		return 0, errors.Trace(reedsolomon.ErrTooFewShards)
+		logger.LogIf(ctx, reedsolomon.ErrTooFewShards)
+		return 0, reedsolomon.ErrTooFewShards
 	}
 
 	// Do we have enough data?
 	if int64(getDataBlockLen(enBlocks, dataBlocks)) < length {
-		return 0, errors.Trace(reedsolomon.ErrShortData)
+		logger.LogIf(ctx, reedsolomon.ErrShortData)
+		return 0, reedsolomon.ErrShortData
 	}
 
 	// Counter to decrement total left to write.
@@ -77,7 +81,8 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
 		if write < int64(len(block)) {
 			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
 			if err != nil {
-				return 0, errors.Trace(err)
+				logger.LogIf(ctx, err)
+				return 0, err
 			}
 			totalWritten += n
 			break
@@ -85,7 +90,8 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
 		// Copy the block.
 		n, err := io.Copy(dst, bytes.NewReader(block))
 		if err != nil {
-			return 0, errors.Trace(err)
+			logger.LogIf(ctx, err)
+			return 0, err
 		}
 
 		// Decrement output size.

cmd/erasure.go

@@ -17,11 +17,12 @@
 package cmd
 
 import (
+	"context"
 	"crypto/subtle"
 	"hash"
 
 	"github.com/klauspost/reedsolomon"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // OfflineDisk represents an unavailable disk.
@@ -44,11 +45,12 @@ type ErasureStorage struct {
 
 // NewErasureStorage creates a new ErasureStorage. The storage erasure codes and protects all data written to
 // the disks.
-func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int, blockSize int64) (s ErasureStorage, err error) {
+func NewErasureStorage(ctx context.Context, disks []StorageAPI, dataBlocks, parityBlocks int, blockSize int64) (s ErasureStorage, err error) {
 	shardsize := (int(blockSize) + dataBlocks - 1) / dataBlocks
 	erasure, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(shardsize))
 	if err != nil {
-		return s, errors.Tracef("failed to create erasure coding: %v", err)
+		logger.LogIf(ctx, err)
+		return s, err
 	}
 	s = ErasureStorage{
 		disks:        make([]StorageAPI, len(disks)),
@@ -62,13 +64,15 @@ func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int, blockSi
 
 // ErasureEncode encodes the given data and returns the erasure-coded data.
 // It returns an error if the erasure coding failed.
-func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) {
+func (s *ErasureStorage) ErasureEncode(ctx context.Context, data []byte) ([][]byte, error) {
 	encoded, err := s.erasure.Split(data)
 	if err != nil {
-		return nil, errors.Tracef("failed to split data: %v", err)
+		logger.LogIf(ctx, err)
+		return nil, err
 	}
 	if err = s.erasure.Encode(encoded); err != nil {
-		return nil, errors.Tracef("failed to encode data: %v", err)
+		logger.LogIf(ctx, err)
+		return nil, err
 	}
 	return encoded, nil
 }
@@ -78,16 +82,17 @@ func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) {
 // It returns an error if the decoding failed.
 func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
 	if err := s.erasure.ReconstructData(data); err != nil {
-		return errors.Tracef("failed to reconstruct data: %v", err)
+		return err
 	}
 	return nil
 }
 
 // ErasureDecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
 // It returns an error if the decoding failed.
-func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(data [][]byte) error {
+func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
 	if err := s.erasure.Reconstruct(data); err != nil {
-		return errors.Tracef("failed to reconstruct data: %v", err)
+		logger.LogIf(ctx, err)
+		return err
 	}
 	return nil
 }

cmd/erasure_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"io"
 	"os"
@@ -52,11 +53,11 @@ func TestErasureDecode(t *testing.T) {
 		copy(buffer, data)
 
 		disks := make([]StorageAPI, test.dataBlocks+test.parityBlocks)
-		storage, err := NewErasureStorage(disks, test.dataBlocks, test.parityBlocks, blockSizeV1)
+		storage, err := NewErasureStorage(context.Background(), disks, test.dataBlocks, test.parityBlocks, blockSizeV1)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create erasure storage: %v", i, err)
 		}
-		encoded, err := storage.ErasureEncode(buffer)
+		encoded, err := storage.ErasureEncode(context.Background(), buffer)
 		if err != nil {
 			t.Fatalf("Test %d: failed to encode data: %v", i, err)
 		}
@@ -69,7 +70,7 @@ func TestErasureDecode(t *testing.T) {
 		}
 
 		if test.reconstructParity {
-			err = storage.ErasureDecodeDataAndParityBlocks(encoded)
+			err = storage.ErasureDecodeDataAndParityBlocks(context.Background(), encoded)
 		} else {
 			err = storage.ErasureDecodeDataBlocks(encoded)
 		}
@@ -98,7 +99,7 @@ func TestErasureDecode(t *testing.T) {
 			}
 
 			decodedData := new(bytes.Buffer)
-			if _, err = writeDataBlocks(decodedData, decoded, test.dataBlocks, 0, int64(len(data))); err != nil {
+			if _, err = writeDataBlocks(context.Background(), decodedData, decoded, test.dataBlocks, 0, int64(len(data))); err != nil {
 				t.Errorf("Test %d: failed to write data blocks: %v", i, err)
 			}
 			if !bytes.Equal(decodedData.Bytes(), data) {