fix timing oracle attack against signature V2/V4 verification (#5335)

This change replaces the non-constant time comparison of
request signatures with a constant time implementation. This
prevents a timing attack which can be used to learn a valid 
signature for a request without knowing the secret key.

Fixes #5334

.github/PULL_REQUEST_TEMPLATE.md

@@ -24,5 +24,6 @@
 - [ ] My code follows the code style of this project.
 - [ ] My change requires a change to the documentation.
 - [ ] I have updated the documentation accordingly.
-- [ ] I have added tests to cover my changes.
+- [ ] I have added unit tests to cover my changes.
+- [ ] I have added/updated functional tests in [mint](https://github.com/minio/mint). (If yes, add `mint` PR # here: )
 - [ ] All new and existing tests passed.

.gitignore

@@ -9,7 +9,7 @@ site/
 /.idea/
 /Minio.iml
 **/access.log
-build
+build/
 vendor/**/*.js
 vendor/**/*.json
 release
@@ -23,3 +23,4 @@ parts/
 prime/
 snap/.snapcraft/
 stage/
+.sia_temp/

Dockerfile

@@ -5,6 +5,7 @@ MAINTAINER Minio Inc <dev@minio.io>
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
+ENV MINIO_UPDATE off
 
 WORKDIR /go/src/github.com/minio/
 

Dockerfile.dev

@@ -5,6 +5,7 @@ MAINTAINER Minio Inc <dev@minio.io>
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
+ENV MINIO_UPDATE off
 
 WORKDIR /go/src/github.com/minio/
 

Dockerfile.release

@@ -4,6 +4,8 @@ MAINTAINER Minio Inc <dev@minio.io>
  
 COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 
+ENV MINIO_UPDATE off
+
 RUN \
      apk add --no-cache ca-certificates && \
      apk add --no-cache --virtual .build-deps curl && \

Makefile

@@ -58,19 +58,17 @@ spelling:
 check: test
 test: verifiers build
 	@echo "Running unit tests"
-	@go test $(GOFLAGS) .
-	@go test $(GOFLAGS) github.com/minio/minio/cmd...
-	@go test $(GOFLAGS) github.com/minio/minio/pkg...
+	@go test $(GOFLAGS) ./...
 	@echo "Verifying build"
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 coverage: build
 	@echo "Running all coverage for minio"
-	@./buildscripts/go-coverage.sh
+	@(env bash $(PWD)/buildscripts/go-coverage.sh)
 
 # Builds minio locally.
 build:
-	@echo "Building minio to $(PWD)/minio ..."
+	@echo "Building minio binary: $(PWD)/minio"
 	@CGO_ENABLED=0 go build --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
 
 pkg-add:
@@ -90,15 +88,9 @@ pkg-list:
 
 # Builds minio and installs it to $GOPATH/bin.
 install: build
-	@echo "Installing minio at $(GOPATH)/bin/minio ..."
+	@echo "Installing minio binary: $(GOPATH)/bin/minio"
 	@cp $(PWD)/minio $(GOPATH)/bin/minio
-	@echo "Check 'minio -h' for help."
-
-release: verifiers
-	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
-
-experimental: verifiers
-	@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
+	@echo "\nInstallation successful. To learn more, try \"minio --help\"."
 
 clean:
 	@echo "Cleaning up all the generated files"

README.md

@@ -26,8 +26,8 @@ Install minio packages using [Homebrew](http://brew.sh/)
 brew install minio/stable/minio
 minio server /data
 ```
-#### Note
-If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
+
+> NOTE: If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
 ```sh
 brew uninstall minio
 brew install minio/stable/minio
@@ -53,18 +53,15 @@ chmod +x minio
 ```
 
 ### Snap
-You can install the latest `minio` [snap](https://snapcraft.io), and help testing the most recent changes of the master branch in [all the supported Linux distros](https://snapcraft.io/docs/core/install) with:
-
+Install minio using [Snap](https://snapcraft.io)
 ```sh
 sudo snap install minio --edge
 ```
 
-Every time a new version of `minio` is pushed to the store, you will get it updated automatically.
-
-You will need to allow the minio snap to observe mounts in the system:
-
+Start minio using `snap run` command
 ```sh
 sudo snap connect minio:mount-observe
+sudo snap run minio server /data
 ```
 
 ## Microsoft Windows
@@ -88,7 +85,6 @@ service minio start
 ```
 
 ## Install from Source
-
 Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
 
 ```sh

README_zh_CN.md

@@ -0,0 +1,116 @@
+# Minio Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+
+Minio 是一个基于Apache License v2.0开源协议的对象存储服务。它兼容亚马逊S3云存储服务接口，非常适合于存储大容量非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。 
+
+Minio是一个非常轻量的服务,可以很简单的和其他应用的结合，类似 NodeJS, Redis 或者 MySQL。
+
+## Docker 容器
+### 稳定版
+```
+docker pull minio/minio
+docker run -p 9000:9000 minio/minio server /data
+```
+
+### 尝鲜版
+```
+docker pull minio/minio:edge
+docker run -p 9000:9000 minio/minio:edge server /data
+```
+更多Docker部署信息请访问 [这里](https://docs.minio.io/docs/minio-docker-quickstart-guide)
+
+## macOS
+### Homebrew
+使用 [Homebrew](http://brew.sh/)安装minio
+
+```sh
+brew install minio/stable/minio
+minio server /data
+```
+#### Note
+如果你之前使用 `brew install minio`安装过minio, 可以用 `minio/stable/minio` 官方镜像进行重装. 由于golang 1.8的bug,homebrew版本不太稳定。
+
+```
+brew uninstall minio
+brew install minio/stable/minio
+```
+
+### 下载二进制文件
+| 操作系统| CPU架构 | 地址|
+| ----------| -------- | ------|
+|Apple macOS|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio |
+```sh
+chmod 755 minio
+./minio server /data
+```
+
+## GNU/Linux
+### 下载二进制文件
+| 操作系统| CPU架构 | 地址|
+| ----------| -------- | ------|
+|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio |
+```sh
+chmod +x minio
+./minio server /data
+```
+
+### 快照版
+你可以下载最新版 `minio` [快照](https://snapcraft.io), 并且帮助我们一起验证master分支上[所有支持的Linux发行版](https://snapcraft.io/docs/core/install) 的一些最新修改:
+
+```sh
+sudo snap install minio --edge
+```
+
+每次有最新的 `minio` 被推送到服务器,你都会自动更新下来.
+
+你需要允许minio snap来观察其安装:
+
+```sh
+sudo snap connect minio:mount-observe
+```
+
+## 微软Windows系统
+### 下载二进制文件
+| 操作系统| CPU架构 | 地址|
+| ----------| -------- | ------|
+|微软Windows系统|64位|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe |
+```sh
+minio.exe server D:\Photos
+```
+
+## FreeBSD
+### Port
+使用 [pkg](https://github.com/freebsd/pkg)进行安装。
+
+```sh
+pkg install minio
+sysrc minio_enable=yes
+sysrc minio_disks=/home/user/Photos
+service minio start
+```
+
+## 使用源码安装
+
+采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
+
+```sh
+go get -u github.com/minio/minio
+```
+
+## 使用Minio浏览器进行验证
+安装后使用浏览器访问[http://127.0.0.1:9000](http://127.0.0.1:9000)，如果可以访问，则表示minio已经安装成功。
+
+![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true)
+
+## 使用Minio客户端 `mc`进行验证
+`mc` 提供了一些UNIX常用命令的替代品，像ls, cat, cp, mirror, diff这些。 它支持文件系统和亚马逊S3云存储服务。 更多信息请参考 [mc快速入门](https://docs.minio.io/docs/minio-client-quickstart-guide) 。
+
+## 了解更多
+- [Minio纠删码入门](https://docs.minio.io/docs/minio-erasure-code-quickstart-guide)
+- [`mc`快速入门](https://docs.minio.io/docs/minio-client-quickstart-guide)
+- [使用 `aws-cli`](https://docs.minio.io/docs/aws-cli-with-minio)
+- [使用 `s3cmd`](https://docs.minio.io/docs/s3cmd-with-minio)
+- [使用 `minio-go` SDK](https://docs.minio.io/docs/golang-client-quickstart-guide)
+- [Minio文档](https://docs.minio.io)
+
+## 如何参与到Minio项目
+请参考 [贡献者指南](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)。欢迎各位中国程序员加到Minio项目中。

appveyor.yml

@@ -38,9 +38,8 @@ test_script:
   # Unit tests
   - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
   - mkdir build\coverage
-  - go test -v -timeout 17m -race github.com/minio/minio/cmd...
-  - go test -v -race github.com/minio/minio/pkg...
-  - go test -v -timeout 17m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
+  - for /f "" %%G in ('go list github.com/minio/minio/... ^| find /i /v "browser/"') do ( go test -v -timeout 20m -race %%G )
+  - go test -v -timeout 20m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
   - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
 
 after_test:

browser/app/index.html

@@ -5,13 +5,13 @@
         <meta charset="UTF-8">
         <meta name="viewport" content="width=device-width, initial-scale=1">
         <title>Minio Browser</title>
-        <link rel="stylesheet" href="/minio/loader.css" type="text/css">
+        <link rel="stylesheet" href="loader.css" type="text/css">
     </head>
 
     <body>
         <div class="page-load">
             <div class="pl-inner">
-                <img src="/minio/logo.svg" alt="">
+                <img src="logo.svg" alt="">
             </div>
         </div>
         <div id="root"></div>
@@ -27,19 +27,19 @@
                     <ul>
                         <li>
                             <a href="http://www.google.com/chrome/">
-                                <img src="/minio/chrome.png" alt="">
+                                <img src="chrome.png" alt="">
                                 <div>Chrome</div>
                             </a>
                         </li>
                         <li>
                             <a href="https://www.mozilla.org/en-US/firefox/new/">
-                                <img src="/minio/firefox.png" alt="">
+                                <img src="firefox.png" alt="">
                                 <div>Firefox</div>
                             </a>
                         </li>
                         <li>
                             <a href="https://www.apple.com/safari/">
-                                <img src="/minio/safari.png" alt="">
+                                <img src="safari.png" alt="">
                                 <div>Safari</div>
                             </a>
                         </li>
@@ -51,6 +51,6 @@
         <![endif]-->
 
         <script>currentUiVersion = 'MINIO_UI_VERSION'</script>
-        <script src="/minio/index_bundle.js"></script>
+        <script src="index_bundle.js"></script>
     </body>
 </html>

browser/app/index.js

@@ -41,6 +41,8 @@ import _Login from './js/components/Login.js'
 import _Browse from './js/components/Browse.js'
 import fontAwesome from 'font-awesome/css/font-awesome.css'
 
+import MaterialDesignIconicFonts from 'material-design-iconic-font/dist/css/material-design-iconic-font.min.css'
+
 import Web from './js/web'
 window.Web = Web
 
@@ -81,7 +83,7 @@ ReactDOM.render((
   <Provider store={ store } web={ web }>
     <Router history={ browserHistory }>
       <Route path='/' component={ App }>
-        <Route path='minio' component={ App }>
+        <Route path={ minioBrowserPrefix } component={ App }>
           <IndexRoute component={ Browse } onEnter={ authNeeded } />
           <Route path='login' component={ Login } onEnter={ authNotNeeded } />
           <Route path=':bucket' component={ Browse } onEnter={ authNeeded } />

browser/app/js/actions.js

@@ -24,6 +24,8 @@ export const SET_CURRENT_BUCKET = 'SET_CURRENT_BUCKET'
 export const SET_CURRENT_PATH = 'SET_CURRENT_PATH'
 export const SET_BUCKETS = 'SET_BUCKETS'
 export const ADD_BUCKET = 'ADD_BUCKET'
+export const REMOVE_BUCKET = 'REMOVE_BUCKET'
+export const SHOW_BUCKET_DROPDOWN = 'SHOW_BUCKET_DROPDOWN'
 export const SET_VISIBLE_BUCKETS = 'SET_VISIBLE_BUCKETS'
 export const SET_OBJECTS = 'SET_OBJECTS'
 export const APPEND_OBJECTS = 'APPEND_OBJECTS'
@@ -173,6 +175,27 @@ export const addBucket = bucket => {
   }
 }
 
+export const removeBucket = bucket => {
+  return {
+    type: REMOVE_BUCKET,
+    bucket
+  }
+}
+
+export const showBucketDropdown = bucket => {
+  return {
+    type: SHOW_BUCKET_DROPDOWN,
+    showBucketDropdown: true
+  }
+}
+
+export const hideBucketDropdown = bucket => {
+  return {
+    type: SHOW_BUCKET_DROPDOWN,
+    showBucketDropdown: false
+  }
+}
+
 export const showMakeBucketModal = () => {
   return {
     type: SHOW_MAKEBUCKET_MODAL,
@@ -314,10 +337,35 @@ export const selectBucket = (newCurrentBucket, prefix) => {
   }
 }
 
+export const deleteBucket = (bucket) => {
+  return (dispatch, getState) => {
+    // DeleteBucket() RPC call will ONLY delete a bucket if it is empty of
+    // objects. This means a call can just be sent, as it is entirely reversable
+    // and won't do any permanent damage.
+    web.DeleteBucket({
+      bucketName: bucket
+    })
+      .then(() => {
+        dispatch(showAlert({
+          type: 'info',
+          message: `Bucket '${bucket}' has been deleted.`
+        }))
+        dispatch(removeBucket(bucket))
+      })
+      .catch(err => {
+        let message = err.message
+        dispatch(showAlert({
+          type: 'danger',
+          message: message
+        }))
+      })
+  }
+}
+
 export const listObjects = () => {
   return (dispatch, getState) => {
-    const {currentBucket, currentPath, marker, objects, istruncated, web} = getState()
-    if (!istruncated) return
+    const {buckets, currentBucket, currentPath, marker, objects, istruncated, web} = getState()
+    if (!istruncated || buckets.length === 0) return
     web.ListObjects({
       bucketName: currentBucket,
       prefix: currentPath,
@@ -465,7 +513,7 @@ export const uploadFile = (file, xhr) => {
   return (dispatch, getState) => {
     const {currentBucket, currentPath} = getState()
     const objectName = `${currentPath}${file.name}`
-    const uploadUrl = `${window.location.origin}/minio/upload/${currentBucket}/${objectName}`
+    const uploadUrl = `${window.location.origin}${minioBrowserPrefix}/upload/${currentBucket}/${objectName}`
     // The slug is a unique identifer for the file upload.
     const slug = `${currentBucket}-${currentPath}-${file.name}`
 
@@ -482,7 +530,7 @@ export const uploadFile = (file, xhr) => {
     }))
 
     xhr.onload = function(event) {
-      if (xhr.status == 401 || xhr.status == 403 || xhr.status == 500) {
+      if (xhr.status == 401 || xhr.status == 403) {
         setShowAbortModal(false)
         dispatch(stopUpload({
           slug
@@ -492,6 +540,16 @@ export const uploadFile = (file, xhr) => {
           message: 'Unauthorized request.'
         }))
       }
+      if (xhr.status == 500) {
+        setShowAbortModal(false)
+        dispatch(stopUpload({
+          slug
+        }))
+        dispatch(showAlert({
+          type: 'danger',
+          message: xhr.responseText
+        }))
+      }
       if (xhr.status == 200) {
         setShowAbortModal(false)
         dispatch(stopUpload({

browser/app/js/components/Browse.js

@@ -157,7 +157,7 @@ export default class Browse extends React.Component {
         // Download the selected file.
         web.CreateURLToken()
           .then(res => {
-            let url = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=${res.token}`
+            let url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encPrefix}?token=${res.token}`
             window.location = url
           })
           .catch(err => dispatch(actions.showAlert({
@@ -210,9 +210,19 @@ export default class Browse extends React.Component {
     dispatch(actions.hideAbout())
   }
 
+  toggleBucketDropdown(e) {
+    const {dispatch, showBucketDropdown} = this.props
+    if (showBucketDropdown) {
+      dispatch(actions.hideBucketDropdown())
+    } else {
+      dispatch(actions.showBucketDropdown())
+    }
+  }
+
   showBucketPolicy(e) {
     e.preventDefault()
     const {dispatch} = this.props
+    this.toggleBucketDropdown(e)
     dispatch(actions.showBucketPolicy())
   }
 
@@ -222,10 +232,17 @@ export default class Browse extends React.Component {
     dispatch(actions.hideBucketPolicy())
   }
 
+  deleteBucket(e, bucket) {
+    e.preventDefault()
+    const {dispatch} = this.props
+    this.toggleBucketDropdown(e)
+    dispatch(actions.deleteBucket(bucket))
+    browserHistory.push(`${minioBrowserPrefix}/`)
+  }
+
   uploadFile(e) {
     e.preventDefault()
-    const {dispatch, buckets} = this.props
-
+    const {dispatch, buckets, currentBucket} = this.props
     if (buckets.length === 0) {
       dispatch(actions.showAlert({
         type: 'danger',
@@ -233,6 +250,13 @@ export default class Browse extends React.Component {
       }))
       return
     }
+    if (currentBucket === '') {
+      dispatch(actions.showAlert({
+        type: 'danger',
+        message: "Please choose a bucket before trying to upload files."
+      }))
+      return
+    }
     let file = e.target.files[0]
     e.target.value = null
     this.xhr = new XMLHttpRequest()
@@ -433,8 +457,7 @@ export default class Browse extends React.Component {
     } else {
       web.CreateURLToken()
         .then(res => {
-          let requestUrl = location.origin + "/minio/zip?token=" + res.token
-
+          let requestUrl = location.origin + minioBrowserPrefix + "/zip?token=" + res.token
           this.xhr = new XMLHttpRequest()
           dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
         })
@@ -503,7 +526,7 @@ export default class Browse extends React.Component {
                                 settingsFunc={ this.showSettings.bind(this) }
                                 logoutFunc={ this.logout.bind(this) } />
     } else {
-      loginButton = <a className='btn btn-danger' href='/minio/login'>Login</a>
+      loginButton = <a className='btn btn-danger' href={minioBrowserPrefix+'/login'}>Login</a>
     }
 
     if (web.LoggedIn()) {
@@ -578,7 +601,9 @@ export default class Browse extends React.Component {
         <SideBar searchBuckets={ this.searchBuckets.bind(this) }
           selectBucket={ this.selectBucket.bind(this) }
           clickOutside={ this.hideSidebar.bind(this) }
-          showPolicy={ this.showBucketPolicy.bind(this) } />
+          showPolicy={ this.showBucketPolicy.bind(this) }
+          deleteBucket={ this.deleteBucket.bind(this) }
+          toggleBucketDropdown={ this.toggleBucketDropdown.bind(this) } />
         <div className="fe-body">
           <div className={ 'list-actions' + (classNames({
                              ' list-actions-toggled': checkedObjects.length > 0

browser/app/js/components/SideBar.js

@@ -21,8 +21,9 @@ import Scrollbars from 'react-custom-scrollbars/lib/Scrollbars'
 import connect from 'react-redux/lib/components/connect'
 
 import logo from '../../img/logo.svg'
+import Dropdown from 'react-bootstrap/lib/Dropdown'
 
-let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchBuckets, sidebarStatus, clickOutside, showPolicy}) => {
+let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchBuckets, sidebarStatus, clickOutside, showPolicy, deleteBucket, toggleBucketDropdown, showBucketDropdown}) => {
 
   const list = visibleBuckets.map((bucket, i) => {
     return <li className={ classNames({
@@ -33,7 +34,19 @@ let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchB
                                     }) }>
                { bucket }
              </a>
-             <i className="fesli-trigger" onClick={ showPolicy }></i>
+             <Dropdown open={bucket === currentBucket && showBucketDropdown} onToggle={toggleBucketDropdown} className="bucket-dropdown" id="bucket-dropdown">
+               <Dropdown.Toggle noCaret>
+                 <i className="zmdi zmdi-more-vert" />
+               </Dropdown.Toggle>
+               <Dropdown.Menu className="dropdown-menu-right">
+                 <li>
+                   <a onClick={ showPolicy }>Edit policy</a>
+                 </li>
+                 <li>
+                   <a onClick={ (e) => deleteBucket(e, bucket) }>Delete</a>
+                 </li>
+               </Dropdown.Menu>
+             </Dropdown>
            </li>
   })
 
@@ -80,6 +93,7 @@ export default connect(state => {
     visibleBuckets: state.visibleBuckets,
     loadBucket: state.loadBucket,
     currentBucket: state.currentBucket,
+    showBucketDropdown: state.showBucketDropdown,
     sidebarStatus: state.sidebarStatus
   }
 })(SideBar)

browser/app/js/constants.js

@@ -17,7 +17,9 @@
 // File for all the browser constants.
 
 // minioBrowserPrefix absolute path.
-export const minioBrowserPrefix = '/minio'
+var p = window.location.pathname
+export const minioBrowserPrefix = p.slice(0, p.indexOf("/", 1))
+
 export const READ_ONLY = 'readonly'
 export const WRITE_ONLY = 'writeonly'
 export const READ_WRITE = 'readwrite'

browser/app/js/reducers.js

@@ -25,6 +25,7 @@ export default (state = {
     storageInfo: {},
     serverInfo: {},
     currentBucket: '',
+    showBucketDropdown: false,
     currentPath: '',
     showMakeBucketModal: false,
     uploads: {},
@@ -71,6 +72,14 @@ export default (state = {
       newState.buckets = [action.bucket, ...newState.buckets]
       newState.visibleBuckets = [action.bucket, ...newState.visibleBuckets]
       break
+    case actions.REMOVE_BUCKET:
+      newState.buckets = newState.buckets.filter(bucket => bucket != action.bucket)
+      newState.visibleBuckets = newState.visibleBuckets.filter(bucket => bucket != action.bucket)
+      newState.currentBucket = ""
+      break
+    case actions.SHOW_BUCKET_DROPDOWN:
+      newState.showBucketDropdown = action.showBucketDropdown
+      break
     case actions.SET_VISIBLE_BUCKETS:
       newState.visibleBuckets = action.visibleBuckets
       break

browser/app/js/web.js

@@ -87,6 +87,9 @@ export default class Web {
   MakeBucket(args) {
     return this.makeCall('MakeBucket', args)
   }
+  DeleteBucket(args) {
+    return this.makeCall('DeleteBucket', args)
+  }
   ListObjects(args) {
     return this.makeCall('ListObjects', args)
   }

browser/app/less/inc/file-explorer.less

@@ -14,7 +14,7 @@
 
 .fe-body {
 	@media(min-width: @screen-md-min) {
-		padding: 0 0 40px @fe-sidebar-width;
+		padding: 0 0 80px @fe-sidebar-width;
 	}
 
 	@media(max-width: @screen-sm-max) {
@@ -79,7 +79,7 @@
 	text-align: center;
 	border: 0;
 	padding: 0;
-	
+
 	span {
 		display: inline-block;
 		height: 100%;

browser/app/less/inc/list.less

@@ -159,7 +159,7 @@ div.fesl-row {
         top: 0;
         width: 35px;
         height: 35px;
-        z-index: 20;
+        z-index: 8;
         opacity: 0;
         cursor: pointer;
 
@@ -223,7 +223,7 @@ div.fesl-row {
         width: 15px;
         height: 15px;
         border: 2px solid @white;
-        z-index: 10;
+        z-index: 7;
         border-radius: 2px;
         top: 10px;
         left: 10px;

browser/app/less/inc/sidebar.less

@@ -111,7 +111,7 @@
 		}
 
 		&:hover {
-			.fesli-trigger {
+			.bucket-dropdown .dropdown-toggle {
 				.opacity(0.6);
 
 				&:hover {
@@ -132,17 +132,40 @@
 	}
 }
 
-.fesli-trigger {
-	.opacity(0);
-	.transition(all);
-	.transition-duration(200ms);
+/* Dropdown */
+.bucket-dropdown {
 	position: absolute;
-	top: 0;
-	right: 0;
+	top: 0px;
+	right: 0px;
 	width: 35px;
 	height: 100%;
 	cursor: pointer;
-	background: url(../../img/more-h-light.svg) no-repeat top 20px left;
+	color: @white;
+
+	.dropdown-toggle {
+		.opacity(0);
+		.transition(all);
+		.transition-duration(200ms);
+		font-size: 20px;
+		background-color: transparent;
+	}
+
+	.dropdown-menu-right {
+		padding: 15px 0;
+		margin-top: -1px;
+		& li {
+			a:before {
+				content: none;
+				-webkit-box-shadow: @dropdown-shadow;
+				box-shadow: @dropdown-shadow;	
+			}
+			&:not(.active):hover {
+				& > a {
+					color: @dropdown-link-hover-color;
+				}
+			}
+		}
+	}
 }
 
 /* Scrollbar */

browser/package.json

@@ -60,6 +60,7 @@
     "humanize": "0.0.9",
     "json-loader": "^0.5.4",
     "local-storage-fallback": "^1.3.0",
+    "material-design-iconic-font": "^2.2.0",
     "mime-db": "^1.25.0",
     "mime-types": "^2.1.13",
     "moment": "^2.15.1",

buildscripts/build.sh

@@ -1,110 +0,0 @@
-#!/bin/bash
-
-_init() {
-    # Save release LDFLAGS
-    LDFLAGS=$(go run buildscripts/gen-ldflags.go)
-
-    # Extract release tag
-    release_tag=$(echo $LDFLAGS | awk {'print $6'} | cut -f2 -d=)
-
-    # Verify release tag.
-    if [ -z "$release_tag" ]; then
-        echo "Release tag cannot be empty. Please check return value of \`go run buildscripts/gen-ldflags.go\`"
-        exit 1;
-    fi
-
-    # Extract release string.
-    release_str=$(echo $MINIO_RELEASE | tr '[:upper:]' '[:lower:]')
-
-    # Verify release string.
-    if [ -z "$release_str" ]; then
-        echo "Release string cannot be empty. Please set \`MINIO_RELEASE\` env variable."
-        exit 1;
-    fi
-
-    # List of supported architectures
-    SUPPORTED_OSARCH='linux/amd64 windows/amd64 darwin/amd64'
-
-    ## System binaries
-    CP=`which cp`
-    SHASUM=`which shasum`
-    SHA256SUM="${SHASUM} -a 256"
-    SED=`which sed`
-}
-
-go_build() {
-    local osarch=$1
-    os=$(echo $osarch | cut -f1 -d'/')
-    arch=$(echo $osarch | cut -f2 -d'/')
-    package=$(go list -f '{{.ImportPath}}')
-    echo -n "-->"
-    printf "%15s:%s\n" "${osarch}" "${package}"
-
-    # Release binary name
-    release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
-    # Release binary downloadable name
-    release_real_bin="$release_str/$os-$arch/$(basename $package)"
-
-    # Release sha1sum name
-    release_shasum="$release_str/$os-$arch/$(basename $package).${release_tag}.shasum"
-    # Release sha1sum default
-    release_shasum_default="$release_str/$os-$arch/$(basename $package).shasum"
-
-    # Release sha256sum name
-    release_sha256sum="$release_str/$os-$arch/$(basename $package).${release_tag}.sha256sum"
-    # Release sha256sum default
-    release_sha256sum_default="$release_str/$os-$arch/$(basename $package).sha256sum"
-
-    # Go build to build the binary.
-    CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
-
-    # Create copy
-    if [ $os == "windows" ]; then
-        $CP -p $release_bin ${release_real_bin}.exe
-    else
-        $CP -p $release_bin $release_real_bin
-    fi
-
-    # Calculate sha1sum
-    shasum_str=$(${SHASUM} ${release_bin})
-    echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
-    $CP -p $release_shasum $release_shasum_default
-
-    # Calculate sha256sum
-    sha256sum_str=$(${SHA256SUM} ${release_bin})
-    echo ${sha256sum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_sha256sum
-    $CP -p $release_sha256sum $release_sha256sum_default
-}
-
-main() {
-    # Build releases.
-    echo "Executing $release_str builds for OS: ${SUPPORTED_OSARCH}"
-    echo  "Choose an OS Arch from the below"
-    for osarch in ${SUPPORTED_OSARCH}; do
-        echo ${osarch}
-    done
-
-    read -p "If you want to build for all, Just press Enter: " chosen_osarch
-    if [ "$chosen_osarch" = "" ] || [ "$chosen_osarch" = "all" ]; then
-        for each_osarch in ${SUPPORTED_OSARCH}; do
-            go_build ${each_osarch}
-        done
-    else
-        local found=0
-        for each_osarch in ${SUPPORTED_OSARCH}; do
-            if [ "$chosen_osarch" = "$each_osarch" ]; then
-                found=1
-            fi
-        done
-        if [ ${found} -eq 1 ]; then
-            go_build ${chosen_osarch}
-        else
-            echo "Unknown architecture \"${chosen_osarch}\""
-            exit 1
-        fi
-    fi
-
-}
-
-# Run main.
-_init && main

buildscripts/go-coverage.sh

@@ -3,8 +3,8 @@
 set -e
 echo "" > coverage.txt
 
-for d in $(go list ./... | grep -v vendor); do
-    go test -coverprofile=profile.out -covermode=atomic $d
+for d in $(go list ./... | grep -v browser); do
+    go test -coverprofile=profile.out -covermode=atomic "$d"
     if [ -f profile.out ]; then
         cat profile.out >> coverage.txt
         rm profile.out

cmd/admin-handlers.go

@@ -28,6 +28,8 @@ import (
 	"strconv"
 	"sync"
 	"time"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 const (
@@ -168,7 +170,7 @@ func (adminAPI adminAPIHandlers) ServiceCredentialsHandler(w http.ResponseWriter
 		return
 	}
 
-	creds, err := createCredential(req.Username, req.Password)
+	creds, err := auth.CreateCredentials(req.Username, req.Password)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -181,12 +183,12 @@ func (adminAPI adminAPIHandlers) ServiceCredentialsHandler(w http.ResponseWriter
 	}
 
 	// Update local credentials in memory.
-	prevCred := serverConfig.SetCredential(creds)
+	prevCred := globalServerConfig.SetCredential(creds)
 
 	// Save credentials to config file
-	if err = serverConfig.Save(); err != nil {
+	if err = globalServerConfig.Save(); err != nil {
 		// Save the current creds when failed to update.
-		serverConfig.SetCredential(prevCred)
+		globalServerConfig.SetCredential(prevCred)
 
 		errorIf(err, "Unable to update the config with new credentials.")
 		writeErrorResponse(w, ErrInternalError, r.URL)
@@ -840,7 +842,7 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
 
 	// Wrap into retrying disks
 	retryingDisks := initRetryableStorageDisks(bootstrapDisks,
-		time.Millisecond, time.Millisecond*5)
+		time.Millisecond, time.Millisecond*5, globalStorageHealthCheckInterval, globalStorageRetryThreshold)
 
 	// Heal format.json on available storage.
 	err = healFormatXL(retryingDisks)
@@ -983,7 +985,7 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	var config serverConfigV19
+	var config serverConfig
 	err = json.Unmarshal(configBytes, &config)
 
 	if err != nil {
@@ -993,7 +995,7 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
 	}
 
 	if globalIsEnvCreds {
-		creds := serverConfig.GetCredential()
+		creds := globalServerConfig.GetCredential()
 		if config.Credential.AccessKey != creds.AccessKey ||
 			config.Credential.SecretKey != creds.SecretKey {
 			writeErrorResponse(w, ErrAdminCredentialsMismatch, r.URL)

cmd/admin-handlers_test.go

@@ -32,6 +32,8 @@ import (
 	"time"
 
 	router "github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/errors"
 )
 
 var configJSON = []byte(`{
@@ -191,7 +193,7 @@ func (atb *adminXLTestBed) TearDown() {
 // initTestObjLayer - Helper function to initialize an XL-based object
 // layer and set globalObjectAPI.
 func initTestXLObjLayer() (ObjectLayer, []string, error) {
-	objLayer, xlDirs, xlErr := prepareXL()
+	objLayer, xlDirs, xlErr := prepareXL16()
 	if xlErr != nil {
 		return nil, nil, xlErr
 	}
@@ -263,7 +265,7 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 
 // getServiceCmdRequest - Constructs a management REST API request for service
 // subcommands for a given cmdType value.
-func getServiceCmdRequest(cmd cmdType, cred credential, body []byte) (*http.Request, error) {
+func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*http.Request, error) {
 	req, err := newTestRequest(cmd.apiMethod(), "/?service", 0, nil)
 	if err != nil {
 		return nil, err
@@ -305,7 +307,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	if cmd == restartCmd {
 		go testServiceSignalReceiver(cmd, t)
 	}
-	credentials := serverConfig.GetCredential()
+	credentials := globalServerConfig.GetCredential()
 	var body []byte
 
 	req, err := getServiceCmdRequest(cmd, credentials, body)
@@ -360,7 +362,7 @@ func TestServiceSetCreds(t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
-	credentials := serverConfig.GetCredential()
+	credentials := globalServerConfig.GetCredential()
 	var body []byte
 
 	testCases := []struct {
@@ -409,7 +411,7 @@ func TestServiceSetCreds(t *testing.T) {
 
 		// If we got 200 OK, check if new credentials are really set
 		if rec.Code == http.StatusOK {
-			cred := serverConfig.GetCredential()
+			cred := globalServerConfig.GetCredential()
 			if cred.AccessKey != testCase.Username {
 				t.Errorf("Test %d: Wrong access key, expected = %s, found = %s", i+1, testCase.Username, cred.AccessKey)
 			}
@@ -486,7 +488,7 @@ func TestListLocksHandler(t *testing.T) {
 		}
 		req.Header.Set(minioAdminOpHeader, "list")
 
-		cred := serverConfig.GetCredential()
+		cred := globalServerConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
@@ -554,7 +556,7 @@ func TestClearLocksHandler(t *testing.T) {
 		}
 		req.Header.Set(minioAdminOpHeader, "clear")
 
-		cred := serverConfig.GetCredential()
+		cred := globalServerConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
@@ -839,7 +841,7 @@ func TestListObjectsHealHandler(t *testing.T) {
 		}
 		req.Header.Set(minioAdminOpHeader, "list-objects")
 
-		cred := serverConfig.GetCredential()
+		cred := globalServerConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to sign list objects needing heal request - %v", i+1, err)
@@ -910,7 +912,7 @@ func TestHealBucketHandler(t *testing.T) {
 
 		req.Header.Set(minioAdminOpHeader, "bucket")
 
-		cred := serverConfig.GetCredential()
+		cred := globalServerConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to sign heal bucket request - %v",
@@ -1013,7 +1015,7 @@ func TestHealObjectHandler(t *testing.T) {
 
 		req.Header.Set(minioAdminOpHeader, "object")
 
-		cred := serverConfig.GetCredential()
+		cred := globalServerConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to sign heal object request - %v", i+1, err)
@@ -1032,15 +1034,15 @@ func buildAdminRequest(queryVal url.Values, opHdr, method string,
 	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {
 	req, err := newTestRequest(method, "/?"+queryVal.Encode(), contentLength, bodySeeker)
 	if err != nil {
-		return nil, traceError(err)
+		return nil, errors.Trace(err)
 	}
 
 	req.Header.Set(minioAdminOpHeader, opHdr)
 
-	cred := serverConfig.GetCredential()
+	cred := globalServerConfig.GetCredential()
 	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 	if err != nil {
-		return nil, traceError(err)
+		return nil, errors.Trace(err)
 	}
 
 	return req, nil

cmd/admin-rpc-client.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -30,6 +29,7 @@ import (
 	"time"
 
 	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/errors"
 )
 
 const (
@@ -139,7 +139,7 @@ func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
 			Version:  Version,
 			CommitID: CommitID,
 			SQSARN:   arns,
-			Region:   serverConfig.GetRegion(),
+			Region:   globalServerConfig.GetRegion(),
 		},
 	}, nil
 }
@@ -158,11 +158,11 @@ func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
 
 // GetConfig - returns config.json of the local server.
 func (lc localAdminClient) GetConfig() ([]byte, error) {
-	if serverConfig == nil {
-		return nil, errors.New("config not present")
+	if globalServerConfig == nil {
+		return nil, fmt.Errorf("config not present")
 	}
 
-	return json.Marshal(serverConfig)
+	return json.Marshal(globalServerConfig)
 }
 
 // GetConfig - returns config.json of the remote server.
@@ -246,7 +246,7 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
 	})
 
 	hostSet := set.CreateStringSet(globalMinioAddr)
-	cred := serverConfig.GetCredential()
+	cred := globalServerConfig.GetCredential()
 	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
 	for _, host := range GetRemotePeers(endpoints) {
 		if hostSet.Contains(host) {
@@ -483,7 +483,7 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
 	configJSON, err := getValidServerConfig(serverConfigs, errs)
 	if err != nil {
 		errorIf(err, "Unable to find a valid server config")
-		return nil, traceError(err)
+		return nil, errors.Trace(err)
 	}
 
 	// Return the config.json that was present quorum or more

cmd/admin-rpc-server.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -26,11 +25,12 @@ import (
 	"time"
 
 	router "github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/errors"
 )
 
 const adminPath = "/admin"
 
-var errUnsupportedBackend = errors.New("not supported for non erasure-code backend")
+var errUnsupportedBackend = fmt.Errorf("not supported for non erasure-code backend")
 
 // adminCmd - exports RPC methods for service status, stop and
 // restart commands.
@@ -104,8 +104,12 @@ func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
 		return err
 	}
 
+	// Wrap into retrying disks
+	retryingDisks := initRetryableStorageDisks(bootstrapDisks,
+		time.Millisecond, time.Millisecond*5, globalStorageHealthCheckInterval, globalStorageRetryThreshold)
+
 	// Initialize new object layer with newly formatted disks.
-	newObjectAPI, err := newXLObjects(bootstrapDisks)
+	newObjectAPI, err := newXLObjects(retryingDisks)
 	if err != nil {
 		return err
 	}
@@ -148,7 +152,7 @@ func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply)
 			Uptime:   UTCNow().Sub(globalBootTime),
 			Version:  Version,
 			CommitID: CommitID,
-			Region:   serverConfig.GetRegion(),
+			Region:   globalServerConfig.GetRegion(),
 			SQSARN:   arns,
 		},
 		StorageInfo: storageInfo,
@@ -165,11 +169,11 @@ func (s *adminCmd) GetConfig(args *AuthRPCArgs, reply *ConfigReply) error {
 		return err
 	}
 
-	if serverConfig == nil {
-		return errors.New("config not present")
+	if globalServerConfig == nil {
+		return fmt.Errorf("config not present")
 	}
 
-	jsonBytes, err := json.Marshal(serverConfig)
+	jsonBytes, err := json.Marshal(globalServerConfig)
 	if err != nil {
 		return err
 	}
@@ -238,7 +242,7 @@ func registerAdminRPCRouter(mux *router.Router) error {
 	adminRPCServer := newRPCServer()
 	err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
 	if err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 	adminRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
 	adminRouter.Path(adminPath).Handler(adminRPCServer)

cmd/admin-rpc-server_test.go

@@ -33,7 +33,7 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
 	}
 	defer os.RemoveAll(rootPath)
 
-	creds := serverConfig.GetCredential()
+	creds := globalServerConfig.GetCredential()
 	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		t.Fatal(err)
@@ -95,7 +95,7 @@ func TestReInitDisks(t *testing.T) {
 	globalIsXL = true
 	adminServer := adminCmd{}
 
-	creds := serverConfig.GetCredential()
+	creds := globalServerConfig.GetCredential()
 	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		t.Fatal(err)
@@ -163,7 +163,7 @@ func TestGetConfig(t *testing.T) {
 	defer os.RemoveAll(rootPath)
 
 	adminServer := adminCmd{}
-	creds := serverConfig.GetCredential()
+	creds := globalServerConfig.GetCredential()
 
 	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
@@ -212,7 +212,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
 	defer os.RemoveAll(rootPath)
 
 	adminServer := adminCmd{}
-	creds := serverConfig.GetCredential()
+	creds := globalServerConfig.GetCredential()
 	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		t.Fatal(err)

cmd/api-errors.go

@@ -20,6 +20,8 @@ import (
 	"encoding/xml"
 	"net/http"
 
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
 )
 
@@ -119,8 +121,20 @@ const (
 	ErrBucketAlreadyExists
 	ErrMetadataTooLarge
 	ErrUnsupportedMetadata
+	ErrMaximumExpires
+	ErrSlowDown
 	// Add new error codes here.
 
+	// Server-Side-Encryption (with Customer provided key) related API errors.
+	ErrInsecureSSECustomerRequest
+	ErrSSEEncryptedObject
+	ErrInvalidEncryptionParameters
+	ErrInvalidSSECustomerAlgorithm
+	ErrInvalidSSECustomerKey
+	ErrMissingSSECustomerKey
+	ErrMissingSSECustomerKeyMD5
+	ErrSSECustomerKeyMD5Mismatch
+
 	// Bucket notification related errors.
 	ErrEventNotification
 	ErrARNNotification
@@ -149,6 +163,11 @@ const (
 	ErrServerNotInitialized
 	ErrOperationTimedOut
 	ErrPartsSizeUnequal
+	ErrInvalidRequest
+
+	// Minio storage class error codes
+	ErrInvalidStorageClass
+
 	// Add new extended error codes here.
 	// Please open a https://github.com/minio/minio/issues before adding
 	// new error codes here.
@@ -158,6 +177,7 @@ const (
 	ErrAdminConfigNoQuorum
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
+	ErrObjectTampered
 )
 
 // error code to APIError structure, these fields carry respective
@@ -178,6 +198,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Unknown metadata directive.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidStorageClass: {
+		Code:           "InvalidStorageClass",
+		Description:    "Invalid storage class.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidRequestBody: {
 		Code:           "InvalidArgument",
 		Description:    "Body shouldn't be set for this request.",
@@ -490,6 +515,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Request is not valid yet",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrSlowDown: {
+		Code:           "SlowDown",
+		Description:    "Please reduce your request",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
 	// FIXME: Actual XML error response also contains the header which missed in list of signed header parameters.
 	ErrUnsignedHeaders: {
 		Code:           "AccessDenied",
@@ -573,6 +603,51 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Range specified is not valid for source object",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMetadataTooLarge: {
+		Code:           "InvalidArgument",
+		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInsecureSSECustomerRequest: {
+		Code:           "InvalidRequest",
+		Description:    errInsecureSSERequest.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSSEEncryptedObject: {
+		Code:           "InvalidRequest",
+		Description:    errEncryptedObject.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidEncryptionParameters: {
+		Code:           "InvalidRequest",
+		Description:    "The encryption parameters are not applicable to this object.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidSSECustomerAlgorithm: {
+		Code:           "InvalidArgument",
+		Description:    errInvalidSSEAlgorithm.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidSSECustomerKey: {
+		Code:           "InvalidArgument",
+		Description:    errInvalidSSEKey.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSSECustomerKey: {
+		Code:           "InvalidArgument",
+		Description:    errMissingSSEKey.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSSECustomerKeyMD5: {
+		Code:           "InvalidArgument",
+		Description:    errMissingSSEKeyMD5.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSSECustomerKeyMD5Mismatch: {
+		Code:           "InvalidArgument",
+		Description:    errSSEKeyMD5Mismatch.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	/// S3 extensions.
 	ErrContentSHA256Mismatch: {
@@ -652,11 +727,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "A timeout occurred while trying to lock a resource",
 		HTTPStatusCode: http.StatusRequestTimeout,
 	},
-	ErrMetadataTooLarge: {
-		Code:           "InvalidArgument",
-		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrUnsupportedMetadata: {
 		Code:           "InvalidArgument",
 		Description:    "Your metadata headers are not supported.",
@@ -667,6 +737,25 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "All parts except the last part should be of the same size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectTampered: {
+		Code:           "XMinioObjectTampered",
+		Description:    errObjectTampered.Error(),
+		HTTPStatusCode: http.StatusPartialContent,
+	},
+	ErrMaximumExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// Generic Invalid-Request error. Should be used for response errors only for unlikely
+	// corner case errors for which introducing new APIErrorCode is not worth it. errorIf()
+	// should be used to log the error at the source of the error for debugging purposes.
+	ErrInvalidRequest: {
+		Code:           "InvalidRequest",
+		Description:    "Invalid Request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
 	// Add your error structure here.
 }
 
@@ -678,7 +767,7 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
-	err = errorCause(err)
+	err = errors.Cause(err)
 	// Verify if the underlying error is signature mismatch.
 	switch err {
 	case errSignatureMismatch:
@@ -687,9 +776,9 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooLarge
 	case errDataTooSmall:
 		apiErr = ErrEntityTooSmall
-	case errInvalidAccessKeyLength:
+	case auth.ErrInvalidAccessKeyLength:
 		apiErr = ErrAdminInvalidAccessKey
-	case errInvalidSecretKeyLength:
+	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
 	}
 
@@ -698,6 +787,27 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return apiErr
 	}
 
+	switch err { // SSE errors
+	case errInsecureSSERequest:
+		return ErrInsecureSSECustomerRequest
+	case errInvalidSSEAlgorithm:
+		return ErrInvalidSSECustomerAlgorithm
+	case errInvalidSSEKey:
+		return ErrInvalidSSECustomerKey
+	case errMissingSSEKey:
+		return ErrMissingSSECustomerKey
+	case errMissingSSEKeyMD5:
+		return ErrMissingSSECustomerKeyMD5
+	case errSSEKeyMD5Mismatch:
+		return ErrSSECustomerKeyMD5Mismatch
+	case errObjectTampered:
+		return ErrObjectTampered
+	case errEncryptedObject:
+		return ErrSSEEncryptedObject
+	case errSSEKeyMismatch:
+		return ErrAccessDenied // no access without correct key
+	}
+
 	switch err.(type) {
 	case StorageFull:
 		apiErr = ErrStorageFull
@@ -763,6 +873,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrUnsupportedMetadata
 	case PartsSizeUnequal:
 		apiErr = ErrPartsSizeUnequal
+	case BucketPolicyNotFound:
+		apiErr = ErrNoSuchBucketPolicy
 	default:
 		apiErr = ErrInternalError
 	}

cmd/api-errors_test.go

@@ -23,116 +23,48 @@ import (
 	"github.com/minio/minio/pkg/hash"
 )
 
+var toAPIErrorCodeTests = []struct {
+	err     error
+	errCode APIErrorCode
+}{
+	{err: hash.BadDigest{}, errCode: ErrBadDigest},
+	{err: hash.SHA256Mismatch{}, errCode: ErrContentSHA256Mismatch},
+	{err: IncompleteBody{}, errCode: ErrIncompleteBody},
+	{err: ObjectExistsAsDirectory{}, errCode: ErrObjectExistsAsDirectory},
+	{err: BucketNameInvalid{}, errCode: ErrInvalidBucketName},
+	{err: BucketExists{}, errCode: ErrBucketAlreadyOwnedByYou},
+	{err: ObjectNotFound{}, errCode: ErrNoSuchKey},
+	{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
+	{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
+	{err: InvalidPart{}, errCode: ErrInvalidPart},
+	{err: InsufficientReadQuorum{}, errCode: ErrReadQuorum},
+	{err: InsufficientWriteQuorum{}, errCode: ErrWriteQuorum},
+	{err: UnsupportedDelimiter{}, errCode: ErrNotImplemented},
+	{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
+	{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
+	{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
+	{err: PartTooSmall{}, errCode: ErrEntityTooSmall},
+	{err: BucketNotEmpty{}, errCode: ErrBucketNotEmpty},
+	{err: BucketNotFound{}, errCode: ErrNoSuchBucket},
+	{err: StorageFull{}, errCode: ErrStorageFull},
+	{err: NotImplemented{}, errCode: ErrNotImplemented},
+	{err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch},
+
+	// SSE-C errors
+	{err: errInsecureSSERequest, errCode: ErrInsecureSSECustomerRequest},
+	{err: errInvalidSSEAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
+	{err: errMissingSSEKey, errCode: ErrMissingSSECustomerKey},
+	{err: errInvalidSSEKey, errCode: ErrInvalidSSECustomerKey},
+	{err: errMissingSSEKeyMD5, errCode: ErrMissingSSECustomerKeyMD5},
+	{err: errSSEKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch},
+	{err: errObjectTampered, errCode: ErrObjectTampered},
+
+	{err: nil, errCode: ErrNone},
+	{err: errors.New("Custom error"), errCode: ErrInternalError}, // Case where err type is unknown.
+}
+
 func TestAPIErrCode(t *testing.T) {
-	testCases := []struct {
-		err     error
-		errCode APIErrorCode
-	}{
-		// Valid cases.
-		{
-			hash.BadDigest{},
-			ErrBadDigest,
-		},
-		{
-			hash.SHA256Mismatch{},
-			ErrContentSHA256Mismatch,
-		},
-		{
-			IncompleteBody{},
-			ErrIncompleteBody,
-		},
-		{
-			ObjectExistsAsDirectory{},
-			ErrObjectExistsAsDirectory,
-		},
-		{
-			BucketNameInvalid{},
-			ErrInvalidBucketName,
-		},
-		{
-			BucketExists{},
-			ErrBucketAlreadyOwnedByYou,
-		},
-		{
-			ObjectNotFound{},
-			ErrNoSuchKey,
-		},
-		{
-			ObjectNameInvalid{},
-			ErrInvalidObjectName,
-		},
-		{
-			InvalidUploadID{},
-			ErrNoSuchUpload,
-		},
-		{
-			InvalidPart{},
-			ErrInvalidPart,
-		},
-		{
-			InsufficientReadQuorum{},
-			ErrReadQuorum,
-		},
-		{
-			InsufficientWriteQuorum{},
-			ErrWriteQuorum,
-		},
-		{
-			UnsupportedDelimiter{},
-			ErrNotImplemented,
-		},
-		{
-			InvalidMarkerPrefixCombination{},
-			ErrNotImplemented,
-		},
-		{
-			InvalidUploadIDKeyCombination{},
-			ErrNotImplemented,
-		},
-		{
-			MalformedUploadID{},
-			ErrNoSuchUpload,
-		},
-		{
-			PartTooSmall{},
-			ErrEntityTooSmall,
-		},
-		{
-			BucketNotEmpty{},
-			ErrBucketNotEmpty,
-		},
-		{
-			BucketNotFound{},
-			ErrNoSuchBucket,
-		},
-		{
-			StorageFull{},
-			ErrStorageFull,
-		},
-		{
-			NotImplemented{},
-			ErrNotImplemented,
-		},
-		{
-			errSignatureMismatch,
-			ErrSignatureDoesNotMatch,
-		}, // End of all valid cases.
-
-		// Case where err is nil.
-		{
-			nil,
-			ErrNone,
-		},
-
-		// Case where err type is unknown.
-		{
-			errors.New("Custom error"),
-			ErrInternalError,
-		},
-	}
-
-	// Validate all the errors with their API error codes.
-	for i, testCase := range testCases {
+	for i, testCase := range toAPIErrorCodeTests {
 		errCode := toAPIErrorCode(testCase.err)
 		if errCode != testCase.errCode {
 			t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)

cmd/api-headers.go

@@ -38,7 +38,7 @@ func setCommonHeaders(w http.ResponseWriter) {
 	w.Header().Set("Server", globalServerUserAgent)
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
-	if region := serverConfig.GetRegion(); region != "" {
+	if region := globalServerConfig.GetRegion(); region != "" {
 		w.Header().Set("X-Amz-Bucket-Region", region)
 	}
 	w.Header().Set("Accept-Ranges", "bytes")

cmd/api-response.go

@@ -567,6 +567,12 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 
 // writeErrorRespone writes error headers
 func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
+	switch errorCode {
+	case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
+		// Set retry-after header to indicate user-agents to retry request after 120secs.
+		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+		w.Header().Set("Retry-After", "120")
+	}
 	apiError := getAPIError(errorCode)
 	// Generate error response.
 	errorResponse := getAPIErrorResponse(apiError, reqURL.Path)

cmd/api-router.go

@@ -16,9 +16,8 @@
 
 package cmd
 
-import (
-	router "github.com/gorilla/mux"
-)
+import router "github.com/gorilla/mux"
+import "net/http"
 
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
@@ -34,70 +33,75 @@ func registerAPIRouter(mux *router.Router) {
 
 	// API Router
 	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
+	var routers []*router.Router
+	if globalDomainName != "" {
+		routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
+	}
+	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
-	// Bucket router
-	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
+	for _, bucket := range routers {
+		// Object operations
+		// HeadObject
+		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
+		// CopyObjectPart
+		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		// PutObjectPart
+		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		// ListObjectPxarts
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
+		// CompleteMultipartUpload
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// NewMultipartUpload
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
+		// AbortMultipartUpload
+		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// GetObject
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
+		// CopyObject
+		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
+		// PutObject
+		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
+		// DeleteObject
+		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
 
-	/// Object operations
-
-	// HeadObject
-	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
-	// CopyObjectPart
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-	// PutObjectPart
-	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-	// ListObjectPxarts
-	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
-	// CompleteMultipartUpload
-	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
-	// NewMultipartUpload
-	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
-	// AbortMultipartUpload
-	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
-	// GetObject
-	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
-	// CopyObject
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
-	// PutObject
-	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
-	// DeleteObject
-	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
-
-	/// Bucket operations
-
-	// GetBucketLocation
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
-	// GetBucketPolicy
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
-	// GetBucketNotification
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
-	// ListenBucketNotification
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
-	// ListMultipartUploads
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
-	// ListObjectsV2
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
-	// ListObjectsV1 (Legacy)
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
-	// PutBucketPolicy
-	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
-	// PutBucketNotification
-	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
-	// PutBucket
-	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
-	// HeadBucket
-	bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
-	// PostPolicy
-	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
-	// DeleteMultipleObjects
-	bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler))
-	// DeleteBucketPolicy
-	bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
-	// DeleteBucket
-	bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+		/// Bucket operations
+		// GetBucketLocation
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
+		// GetBucketPolicy
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+		// GetBucketNotification
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
+		// ListenBucketNotification
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
+		// ListMultipartUploads
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
+		// ListObjectsV2
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
+		// ListObjectsV1 (Legacy)
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
+		// PutBucketPolicy
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
+		// PutBucketNotification
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
+		// PutBucket
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
+		// HeadBucket
+		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
+		// PostPolicy
+		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
+		// DeleteMultipleObjects
+		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
+		// DeleteBucketPolicy
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
+		// DeleteBucket
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+	}
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods("GET").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+	apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+
+	// If none of the routes match.
+	apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
 }

cmd/auth-handler.go

@@ -127,7 +127,11 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string)
 	if reqAuthType == authTypeAnonymous && policyAction != "" {
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 		sourceIP := getSourceIPAddress(r)
-		return enforceBucketPolicy(bucket, policyAction, r.URL.Path,
+		resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
+		if err != nil {
+			return ErrInternalError
+		}
+		return enforceBucketPolicy(bucket, policyAction, resource,
 			r.Referer(), sourceIP, r.URL.Query())
 	}
 

cmd/auth-handler_test.go

@@ -23,6 +23,8 @@ import (
 	"net/url"
 	"os"
 	"testing"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Test get request auth type.
@@ -302,7 +304,7 @@ func mustNewRequest(method string, urlStr string, contentLength int64, body io.R
 // is signed with AWS Signature V4, fails if not able to do so.
 func mustNewSignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
-	cred := serverConfig.GetCredential()
+	cred := globalServerConfig.GetCredential()
 	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
 		t.Fatalf("Unable to inititalized new signed http request %s", err)
 	}
@@ -312,7 +314,7 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod
 func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	req.Header.Set("Content-Md5", "YWFhYWFhYWFhYWFhYWFhCg==")
-	cred := serverConfig.GetCredential()
+	cred := globalServerConfig.GetCredential()
 	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
 		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
@@ -327,12 +329,12 @@ func TestIsReqAuthenticated(t *testing.T) {
 	}
 	defer os.RemoveAll(path)
 
-	creds, err := createCredential("myuser", "mypassword")
+	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
 		t.Fatalf("unable create credential, %s", err)
 	}
 
-	serverConfig.SetCredential(creds)
+	globalServerConfig.SetCredential(creds)
 
 	// List of test cases for validating http request authentication.
 	testCases := []struct {
@@ -351,7 +353,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	// Validates all testcases.
 	for _, testCase := range testCases {
-		if s3Error := isReqAuthenticated(testCase.req, serverConfig.GetRegion()); s3Error != testCase.s3Error {
+		if s3Error := isReqAuthenticated(testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
 			t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
 		}
 	}

cmd/auth-rpc-server_test.go

@@ -28,7 +28,7 @@ func TestLogin(t *testing.T) {
 		t.Fatalf("Failed to create test config - %v", err)
 	}
 	defer os.RemoveAll(rootPath)
-	creds := serverConfig.GetCredential()
+	creds := globalServerConfig.GetCredential()
 	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		t.Fatal(err)

cmd/browser-peer-rpc.go

@@ -21,6 +21,8 @@ import (
 	"path"
 	"sync"
 	"time"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // SetAuthPeerArgs - Arguments collection for SetAuth RPC call
@@ -29,7 +31,7 @@ type SetAuthPeerArgs struct {
 	AuthRPCArgs
 
 	// New credentials that receiving peer should update to.
-	Creds credential
+	Creds auth.Credentials
 }
 
 // SetAuthPeer - Update to new credentials sent from a peer Minio
@@ -49,12 +51,12 @@ func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthR
 	}
 
 	// Update credentials in memory
-	prevCred := serverConfig.SetCredential(args.Creds)
+	prevCred := globalServerConfig.SetCredential(args.Creds)
 
 	// Save credentials to config file
-	if err := serverConfig.Save(); err != nil {
+	if err := globalServerConfig.Save(); err != nil {
 		// Save the current creds when failed to update.
-		serverConfig.SetCredential(prevCred)
+		globalServerConfig.SetCredential(prevCred)
 
 		errorIf(err, "Unable to update the config with new credentials sent from browser RPC.")
 		return err
@@ -64,7 +66,7 @@ func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthR
 }
 
 // Sends SetAuthPeer RPCs to all peers in the Minio cluster
-func updateCredsOnPeers(creds credential) map[string]error {
+func updateCredsOnPeers(creds auth.Credentials) map[string]error {
 	// Get list of peer addresses (from globalS3Peers)
 	peers := []string{}
 	for _, p := range globalS3Peers {
@@ -75,7 +77,7 @@ func updateCredsOnPeers(creds credential) map[string]error {
 	errs := make([]error, len(peers))
 	var wg sync.WaitGroup
 
-	serverCred := serverConfig.GetCredential()
+	serverCred := globalServerConfig.GetCredential()
 	// Launch go routines to send request to each peer in parallel.
 	for ix := range peers {
 		wg.Add(1)

cmd/browser-peer-rpc_test.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"path"
 	"testing"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // API suite container common to both FS and XL.
@@ -61,7 +63,7 @@ func TestBrowserPeerRPC(t *testing.T) {
 // Tests for browser peer rpc.
 func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 	// Construct RPC call arguments.
-	creds, err := createCredential("abcd1", "abcd1234")
+	creds, err := auth.CreateCredentials("abcd1", "abcd1234")
 	if err != nil {
 		t.Fatalf("unable to create credential. %v", err)
 	}

cmd/browser-rpc-router.go

@@ -18,6 +18,8 @@ package cmd
 
 import (
 	router "github.com/gorilla/mux"
+
+	"github.com/minio/minio/pkg/errors"
 )
 
 // Set up an RPC endpoint that receives browser related calls. The
@@ -40,7 +42,7 @@ func registerBrowserPeerRPCRouter(mux *router.Router) error {
 	bpRPCServer := newRPCServer()
 	err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)
 	if err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	bpRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()

cmd/bucket-handlers-listobjects.go

@@ -89,7 +89,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}

cmd/bucket-handlers.go

@@ -24,11 +24,15 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"path/filepath"
+	"reflect"
 	"strings"
 	"sync"
 
 	mux "github.com/gorilla/mux"
+	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
 )
 
@@ -37,7 +41,7 @@ import (
 func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, queryParams url.Values) (s3Error APIErrorCode) {
 	// Verify if bucket actually exists
 	if err := checkBucketExist(bucket, newObjectLayerFn()); err != nil {
-		err = errorCause(err)
+		err = errors.Cause(err)
 		switch err.(type) {
 		case BucketNameInvalid:
 			// Return error for invalid bucket name.
@@ -56,8 +60,8 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que
 	}
 
 	// Fetch bucket policy, if policy is not set return access denied.
-	policy := globalBucketPolicies.GetBucketPolicy(bucket)
-	if policy == nil {
+	p := globalBucketPolicies.GetBucketPolicy(bucket)
+	if reflect.DeepEqual(p, emptyBucketPolicy) {
 		return ErrAccessDenied
 	}
 
@@ -65,7 +69,7 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que
 	arn := bucketARNPrefix + strings.TrimSuffix(strings.TrimPrefix(resource, "/"), "/")
 
 	// Get conditions for policy verification.
-	conditionKeyMap := make(map[string]set.StringSet)
+	conditionKeyMap := make(policy.ConditionKeyMap)
 	for queryParam := range queryParams {
 		conditionKeyMap[queryParam] = set.CreateStringSet(queryParams.Get(queryParam))
 	}
@@ -78,7 +82,7 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que
 	conditionKeyMap["ip"] = set.CreateStringSet(sourceIP)
 
 	// Validate action, resource and conditions with current policy statements.
-	if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, policy.Statements) {
+	if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, p.Statements) {
 		return ErrAccessDenied
 	}
 	return ErrNone
@@ -90,14 +94,14 @@ func isBucketActionAllowed(action, bucket, prefix string) bool {
 		return false
 	}
 
-	policy := globalBucketPolicies.GetBucketPolicy(bucket)
-	if policy == nil {
+	bp := globalBucketPolicies.GetBucketPolicy(bucket)
+	if reflect.DeepEqual(bp, emptyBucketPolicy) {
 		return false
 	}
 	resource := bucketARNPrefix + path.Join(bucket, prefix)
 	var conditionKeyMap map[string]set.StringSet
 	// Validate action, resource and conditions with current policy statements.
-	return bucketPolicyEvalStatements(action, resource, conditionKeyMap, policy.Statements)
+	return bucketPolicyEvalStatements(action, resource, conditionKeyMap, bp.Statements)
 }
 
 // GetBucketLocationHandler - GET Bucket location.
@@ -116,7 +120,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 	s3Error := checkRequestAuthType(r, bucket, "s3:GetBucketLocation", globalMinioDefaultRegion)
 	if s3Error == ErrInvalidRegion {
 		// Clients like boto3 send getBucketLocation() call signed with region that is configured.
-		s3Error = checkRequestAuthType(r, "", "s3:GetBucketLocation", serverConfig.GetRegion())
+		s3Error = checkRequestAuthType(r, "", "s3:GetBucketLocation", globalServerConfig.GetRegion())
 	}
 	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
@@ -132,7 +136,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 	// Generate response.
 	encodedSuccessResponse := encodeResponse(LocationResponse{})
 	// Get current region.
-	region := serverConfig.GetRegion()
+	region := globalServerConfig.GetRegion()
 	if region != globalMinioDefaultRegion {
 		encodedSuccessResponse = encodeResponse(LocationResponse{
 			Location: region,
@@ -161,7 +165,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucketMultipartUploads", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucketMultipartUploads", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -208,7 +212,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	s3Error := checkRequestAuthType(r, "", "", globalMinioDefaultRegion)
 	if s3Error == ErrInvalidRegion {
 		// Clients like boto3 send listBuckets() call signed with region that is configured.
-		s3Error = checkRequestAuthType(r, "", "", serverConfig.GetRegion())
+		s3Error = checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
 	}
 	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
@@ -242,7 +246,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	var authError APIErrorCode
-	if authError = checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); authError != ErrNone {
+	if authError = checkRequestAuthType(r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); authError != ErrNone {
 		// In the event access is denied, a 200 response should still be returned
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 		if authError != ErrAccessDenied {
@@ -325,7 +329,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			deletedObjects = append(deletedObjects, object)
 			continue
 		}
-		if _, ok := errorCause(err).(ObjectNotFound); ok {
+		if _, ok := errors.Cause(err).(ObjectNotFound); ok {
 			// If the object is not found it should be
 			// accounted as deleted as per S3 spec.
 			deletedObjects = append(deletedObjects, object)
@@ -381,7 +385,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// PutBucket does not have any bucket action.
-	s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion())
+	s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
 	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
@@ -436,12 +440,24 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
+	bucket := mux.Vars(r)["bucket"]
+
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
 		writeErrorResponse(w, ErrMissingContentLength, r.URL)
 		return
 	}
+	resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
+	if err != nil {
+		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		return
+	}
+	// Make sure that the URL  does not contain object name.
+	if bucket != filepath.Clean(resource[1:]) {
+		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
 
 	// Here the parameter is the size of the form data that should
 	// be loaded in memory, the remaining being put in temporary files.
@@ -480,7 +496,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Close multipart file
 	defer fileBody.Close()
 
-	bucket := mux.Vars(r)["bucket"]
 	formValues.Set("Bucket", bucket)
 
 	if fileName != "" && strings.Contains(formValues.Get("Key"), "${filename}") {
@@ -634,7 +649,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponseHeadersOnly(w, s3Error)
 		return
 	}
@@ -664,7 +679,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 
 	// DeleteBucket does not have any bucket action.
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -690,7 +705,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	_ = removeBucketPolicy(bucket, objectAPI)
 
 	// Notify all peers (including self) to update in-memory state
-	S3PeersUpdateBucketPolicy(bucket, policyChange{true, nil})
+	S3PeersUpdateBucketPolicy(bucket, policyChange{true, policy.BucketAccessPolicy{}})
 
 	// Delete notification config, if present - ignore any errors.
 	_ = removeNotificationConfig(bucket, objectAPI)

cmd/bucket-handlers_test.go

@@ -24,6 +24,8 @@ import (
 	"net/http/httptest"
 	"strconv"
 	"testing"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Wrapper for calling GetBucketPolicy HTTP handler tests for both XL multiple disks and single node setup.
@@ -32,7 +34,7 @@ func TestGetBucketLocationHandler(t *testing.T) {
 }
 
 func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	initBucketPolicies(obj)
 
 	// test cases with sample input and expected output.
@@ -177,7 +179,7 @@ func TestHeadBucketHandler(t *testing.T) {
 }
 
 func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	initBucketPolicies(obj)
 
 	// test cases with sample input and expected output.
@@ -284,7 +286,7 @@ func TestListMultipartUploadsHandler(t *testing.T) {
 
 // testListMultipartUploadsHandler - Tests validate listing of multipart uploads.
 func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	initBucketPolicies(obj)
 
 	// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
@@ -522,7 +524,7 @@ func TestListBucketsHandler(t *testing.T) {
 
 // testListBucketsHandler - Tests validate listing of buckets.
 func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 
 	testCases := []struct {
 		bucketName         string
@@ -615,7 +617,7 @@ func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
 }
 
 func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	initBucketPolicies(obj)
 
 	var err error
@@ -805,7 +807,7 @@ func TestIsBucketActionAllowed(t *testing.T) {
 }
 
 func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 
 	testCases := []struct {
 		// input.

cmd/bucket-notification-datatypes.go

@@ -137,13 +137,13 @@ type bucketMeta struct {
 
 // Notification event object metadata.
 type objectMeta struct {
-	Key         string            `json:"key"`
-	Size        int64             `json:"size,omitempty"`
-	ETag        string            `json:"eTag,omitempty"`
-	ContentType string            `json:"contentType,omitempty"`
-	UserDefined map[string]string `json:"userDefined,omitempty"`
-	VersionID   string            `json:"versionId,omitempty"`
-	Sequencer   string            `json:"sequencer"`
+	Key          string            `json:"key"`
+	Size         int64             `json:"size,omitempty"`
+	ETag         string            `json:"eTag,omitempty"`
+	ContentType  string            `json:"contentType,omitempty"`
+	UserMetadata map[string]string `json:"userMetadata,omitempty"`
+	VersionID    string            `json:"versionId,omitempty"`
+	Sequencer    string            `json:"sequencer"`
 }
 
 const (
@@ -209,5 +209,5 @@ type arnSQS struct {
 
 // Stringer for constructing AWS ARN compatible string.
 func (m arnSQS) String() string {
-	return minioSqs + serverConfig.GetRegion() + ":" + m.AccountID + ":" + m.Type
+	return minioSqs + globalServerConfig.GetRegion() + ":" + m.AccountID + ":" + m.Type
 }

cmd/bucket-notification-handlers.go

@@ -22,10 +22,13 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
+	"syscall"
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/errors"
 )
 
 const (
@@ -46,7 +49,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -63,13 +66,13 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	// Attempt to successfully load notification config.
 	nConfig, err := loadNotificationConfig(bucket, objAPI)
-	if err != nil && errorCause(err) != errNoSuchNotifications {
+	if err != nil && errors.Cause(err) != errNoSuchNotifications {
 		errorIf(err, "Unable to read notification configuration.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 	// For no notifications we write a dummy XML.
-	if errorCause(err) == errNoSuchNotifications {
+	if errors.Cause(err) == errNoSuchNotifications {
 		// Complies with the s3 behavior in this regard.
 		nConfig = &notificationConfig{}
 	}
@@ -100,7 +103,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -150,6 +153,12 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Convert the incoming ARNs properly to the GetRegion().
+	for i, queueConfig := range notificationCfg.QueueConfigs {
+		queueConfig.QueueARN = unmarshalSqsARN(queueConfig.QueueARN).String()
+		notificationCfg.QueueConfigs[i] = queueConfig
+	}
+
 	// Put bucket notification config.
 	err = PutBucketNotificationConfig(bucket, &notificationCfg, objectAPI)
 	if err != nil {
@@ -207,14 +216,6 @@ func writeNotification(w http.ResponseWriter, notification map[string][]Notifica
 		return err
 	}
 
-	// https://github.com/containous/traefik/issues/560
-	// https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
-	//
-	// Proxies might buffer the connection to avoid this we
-	// need the proper MIME type before writing to client.
-	// This MIME header tells the proxies to avoid buffering
-	w.Header().Set("Content-Type", "text/event-stream")
-
 	// Add additional CRLF characters for client to
 	// differentiate the individual events properly.
 	_, err = w.Write(append(notificationBytes, crlf...))
@@ -226,25 +227,61 @@ func writeNotification(w http.ResponseWriter, notification map[string][]Notifica
 // CRLF character used for chunked transfer in accordance with HTTP standards.
 var crlf = []byte("\r\n")
 
-// sendBucketNotification - writes notification back to client on the response writer
-// for each notification input, otherwise writes whitespace characters periodically
-// to keep the connection active. Each notification messages are terminated by CRLF
-// character. Upon any error received on response writer the for loop exits.
-func sendBucketNotification(w http.ResponseWriter, arnListenerCh <-chan []NotificationEvent) {
-	var dummyEvents = map[string][]NotificationEvent{"Records": nil}
-	// Continuously write to client either timely empty structures
-	// every 5 seconds, or return back the notifications.
+// listenChan A `listenChan` provides a data channel to send event
+// notifications on and `doneCh` to signal that events are no longer
+// being received. It also sends empty events (whitespace) to keep the
+// underlying connection alive.
+type listenChan struct {
+	doneCh chan struct{}
+	dataCh chan []NotificationEvent
+}
+
+// newListenChan returns a listenChan with properly initialized
+// unbuffered channels.
+func newListenChan() *listenChan {
+	return &listenChan{
+		doneCh: make(chan struct{}),
+		dataCh: make(chan []NotificationEvent),
+	}
+}
+
+// sendNotificationEvent sends notification events on the data channel
+// unless doneCh is not closed
+func (l *listenChan) sendNotificationEvent(events []NotificationEvent) {
+	select {
+	// Returns immediately if receiver has quit.
+	case <-l.doneCh:
+	// Blocks until receiver is available.
+	case l.dataCh <- events:
+	}
+}
+
+// waitForListener writes event notification OR whitespaces on
+// ResponseWriter until client closes connection
+func (l *listenChan) waitForListener(w http.ResponseWriter) {
+
+	// Logs errors other than EPIPE and ECONNRESET.
+	// EPIPE and ECONNRESET indicate that the client stopped
+	// listening to notification events.
+	logClientError := func(err error, msg string) {
+		if oe, ok := err.(*net.OpError); ok && (oe.Err == syscall.EPIPE || oe.Err ==
+			syscall.ECONNRESET) {
+			errorIf(err, msg)
+		}
+	}
+
+	emptyEvent := map[string][]NotificationEvent{"Records": nil}
+	defer close(l.doneCh)
 	for {
 		select {
-		case events := <-arnListenerCh:
+		case events := <-l.dataCh:
 			if err := writeNotification(w, map[string][]NotificationEvent{"Records": events}); err != nil {
-				errorIf(err, "Unable to write notification to client.")
+				logClientError(err, "Unable to write notification")
 				return
 			}
-		case <-time.After(globalSNSConnAlive): // Wait for global conn active seconds.
-			if err := writeNotification(w, dummyEvents); err != nil {
-				// FIXME - do not log for all errors.
-				errorIf(err, "Unable to write notification to client.")
+		case <-time.After(globalSNSConnAlive):
+			if err := writeNotification(w, emptyEvent); err != nil {
+				logClientError(err, "Unable to write empty notification")
 				return
 			}
 		}
@@ -260,7 +297,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -301,7 +338,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	accountARN := fmt.Sprintf(
 		"%s:%s:%s:%s-%s",
 		minioTopic,
-		serverConfig.GetRegion(),
+		globalServerConfig.GetRegion(),
 		accountID,
 		snsTypeMinio,
 		targetServer,
@@ -340,12 +377,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		},
 	}
 
-	// Setup a listening channel that will receive notifications
-	// from the RPC handler.
-	nEventCh := make(chan []NotificationEvent)
-	defer close(nEventCh)
+	// Setup a listen channel to receive notifications like
+	// s3:ObjectCreated, s3:ObjectDeleted etc.
+	nListenCh := newListenChan()
 	// Add channel for listener events
-	if err = globalEventNotifier.AddListenerChan(accountARN, nEventCh); err != nil {
+	if err = globalEventNotifier.AddListenerChan(accountARN, nListenCh); err != nil {
 		errorIf(err, "Error adding a listener!")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -355,8 +391,8 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	defer globalEventNotifier.RemoveListenerChan(accountARN)
 
 	// Update topic config to bucket config and persist - as soon
-	// as this call compelets, events may start appearing in
-	// nEventCh
+	// as this call completes, events may start appearing in
+	// nListenCh
 	lc := listenerConfig{
 		TopicConfig:  *topicCfg,
 		TargetServer: targetServer,
@@ -372,8 +408,16 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	// Add all common headers.
 	setCommonHeaders(w)
 
-	// Start sending bucket notifications.
-	sendBucketNotification(w, nEventCh)
+	// https://github.com/containous/traefik/issues/560
+	// https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
+	//
+	// Proxies might buffer the connection to avoid this we
+	// need the proper MIME type before writing to client.
+	// This MIME header tells the proxies to avoid buffering
+	w.Header().Set("Content-Type", "text/event-stream")
+
+	// Start writing bucket notifications to ResponseWriter.
+	nListenCh.waitForListener(w)
 }
 
 // AddBucketListenerConfig - Updates on disk state of listeners, and

cmd/bucket-notification-handlers_test.go

@@ -21,13 +21,19 @@ import (
 	"bytes"
 	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"reflect"
+	"sync"
 	"testing"
+
+	"time"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Implement a dummy flush writer.
@@ -111,7 +117,47 @@ func TestWriteNotification(t *testing.T) {
 	}
 }
 
-func TestSendBucketNotification(t *testing.T) {
+// testResponseWriter implements `http.ResponseWriter` that buffers
+// response body in a `bytes.Buffer` and returns error after `failCount`
+// calls to `Write` method
+type testResponseWriter struct {
+	mu        sync.Mutex
+	failCount int
+	buf       *bytes.Buffer
+	m         http.Header
+}
+
+func newTestResponseWriter(failAt int) *testResponseWriter {
+	return &testResponseWriter{
+		buf:       new(bytes.Buffer),
+		m:         make(http.Header),
+		failCount: failAt,
+	}
+}
+
+func (trw *testResponseWriter) Flush() {
+}
+
+func (trw *testResponseWriter) Write(p []byte) (int, error) {
+	trw.mu.Lock()
+	defer trw.mu.Unlock()
+
+	if trw.failCount == 0 {
+		return 0, errors.New("Custom error")
+	}
+	trw.failCount--
+
+	return trw.buf.Write(p)
+}
+
+func (trw *testResponseWriter) Header() http.Header {
+	return trw.m
+}
+
+func (trw *testResponseWriter) WriteHeader(i int) {
+}
+
+func TestListenChan(t *testing.T) {
 	// Initialize a new test config.
 	root, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
@@ -119,18 +165,8 @@ func TestSendBucketNotification(t *testing.T) {
 	}
 	defer os.RemoveAll(root)
 
-	eventCh := make(chan []NotificationEvent)
-
-	// Create a Pipe with FlushWriter on the write-side and bufio.Scanner
-	// on the reader-side to receive notification over the listen channel in a
-	// synchronized manner.
-	pr, pw := io.Pipe()
-	fw := newFlushWriter(pw)
-	scanner := bufio.NewScanner(pr)
-	// Start a go-routine to wait for notification events.
-	go func(listenerCh <-chan []NotificationEvent) {
-		sendBucketNotification(fw, listenerCh)
-	}(eventCh)
+	// Create a listen channel to manage notifications
+	nListenCh := newListenChan()
 
 	// Construct notification events to be passed on the events channel.
 	var events []NotificationEvent
@@ -140,37 +176,68 @@ func TestSendBucketNotification(t *testing.T) {
 		ObjectCreatedCopy,
 		ObjectCreatedCompleteMultipartUpload,
 	}
+
 	for _, evType := range evTypes {
 		events = append(events, newNotificationEvent(eventData{
 			Type: evType,
 		}))
 	}
-	// Send notification events to the channel on which sendBucketNotification
-	// is waiting on.
-	eventCh <- events
 
-	// Read from the pipe connected to the ResponseWriter.
-	scanner.Scan()
-	notificationBytes := scanner.Bytes()
-
-	// Close the read-end and send an empty notification event on the channel
-	// to signal sendBucketNotification to terminate.
-	pr.Close()
-	eventCh <- []NotificationEvent{}
-	close(eventCh)
-
-	// Checking if the notification are the same as those sent over the channel.
-	var notifications map[string][]NotificationEvent
-	err = json.Unmarshal(notificationBytes, &notifications)
-	if err != nil {
-		t.Fatal("Failed to Unmarshal notification")
-	}
-	records := notifications["Records"]
-	for i, rec := range records {
-		if rec.EventName == evTypes[i].String() {
-			continue
+	// Send notification events one-by-one
+	go func() {
+		for _, event := range events {
+			nListenCh.sendNotificationEvent([]NotificationEvent{event})
 		}
-		t.Errorf("Failed to receive %d event %s", i, evTypes[i].String())
+	}()
+
+	// Create a http.ResponseWriter that fails after len(events)
+	// number of times
+	trw := newTestResponseWriter(len(events))
+
+	// Wait for all (4) notification events to be received
+	nListenCh.waitForListener(trw)
+
+	// Used to read JSON-formatted event stream line-by-line
+	scanner := bufio.NewScanner(trw.buf)
+	var records map[string][]NotificationEvent
+	for i := 0; scanner.Scan(); i++ {
+		err = json.Unmarshal(scanner.Bytes(), &records)
+		if err != nil {
+			t.Fatalf("Failed to unmarshal json %v", err)
+		}
+
+		nEvent := records["Records"][0]
+		if nEvent.EventName != evTypes[i].String() {
+			t.Errorf("notification event name mismatch, expected %s but got %s", evTypes[i], nEvent.EventName)
+		}
+	}
+}
+
+func TestSendNotificationEvent(t *testing.T) {
+	// This test verifies that sendNotificationEvent function
+	// returns once listenChan.doneCh is closed
+
+	l := newListenChan()
+	testCh := make(chan struct{})
+	timeout := 5 * time.Second
+
+	go func() {
+		// Send one empty notification event on listenChan
+		events := []NotificationEvent{NotificationEvent{}}
+		l.sendNotificationEvent(events)
+		testCh <- struct{}{}
+	}()
+
+	// close l.doneCh to signal client exiting from
+	// ListenBucketNotification API call
+	close(l.doneCh)
+
+	select {
+	case <-time.After(timeout):
+		t.Fatalf("sendNotificationEvent didn't return after %v seconds", timeout)
+	case <-testCh:
+		// If we reach this case, sendNotificationEvent
+		// returned on closing l.doneCh
 	}
 }
 
@@ -181,7 +248,7 @@ func TestGetBucketNotificationHandler(t *testing.T) {
 }
 
 func testGetBucketNotificationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	// declare sample configs
 	filterRules := []filterRule{
 		{
@@ -254,7 +321,7 @@ func TestPutBucketNotificationHandler(t *testing.T) {
 }
 
 func testPutBucketNotificationHandler(obj ObjectLayer, instanceType,
-	bucketName string, apiRouter http.Handler, credentials credential,
+	bucketName string, apiRouter http.Handler, credentials auth.Credentials,
 	t *testing.T) {
 
 	// declare sample configs
@@ -344,7 +411,7 @@ func TestListenBucketNotificationNilHandler(t *testing.T) {
 }
 
 func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	// get random bucket name.
 	randBucket := getRandomBucketName()
 
@@ -371,7 +438,7 @@ func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucke
 }
 
 func testRemoveNotificationConfig(obj ObjectLayer, instanceType,
-	bucketName string, apiRouter http.Handler, credentials credential,
+	bucketName string, apiRouter http.Handler, credentials auth.Credentials,
 	t *testing.T) {
 
 	invalidBucket := "Invalid\\Bucket"

cmd/bucket-notification-utils.go

@@ -116,9 +116,13 @@ func checkARN(arn, arnType string) APIErrorCode {
 	if len(strs) != 6 {
 		return ErrARNNotification
 	}
-	if serverConfig.GetRegion() != "" {
+
+	// Server region is allowed to be empty by default,
+	// in such a scenario ARN region is not validating
+	// allowing all regions.
+	if sregion := globalServerConfig.GetRegion(); sregion != "" {
 		region := strs[3]
-		if region != serverConfig.GetRegion() {
+		if region != sregion {
 			return ErrRegionNotification
 		}
 	}
@@ -142,34 +146,34 @@ func isValidQueueID(queueARN string) bool {
 	// Is Queue identifier valid?.
 
 	if isAMQPQueue(sqsARN) { // AMQP eueue.
-		amqpN := serverConfig.Notify.GetAMQPByID(sqsARN.AccountID)
+		amqpN := globalServerConfig.Notify.GetAMQPByID(sqsARN.AccountID)
 		return amqpN.Enable && amqpN.URL != ""
 	} else if isMQTTQueue(sqsARN) {
-		mqttN := serverConfig.Notify.GetMQTTByID(sqsARN.AccountID)
+		mqttN := globalServerConfig.Notify.GetMQTTByID(sqsARN.AccountID)
 		return mqttN.Enable && mqttN.Broker != ""
 	} else if isNATSQueue(sqsARN) {
-		natsN := serverConfig.Notify.GetNATSByID(sqsARN.AccountID)
+		natsN := globalServerConfig.Notify.GetNATSByID(sqsARN.AccountID)
 		return natsN.Enable && natsN.Address != ""
 	} else if isElasticQueue(sqsARN) { // Elastic queue.
-		elasticN := serverConfig.Notify.GetElasticSearchByID(sqsARN.AccountID)
+		elasticN := globalServerConfig.Notify.GetElasticSearchByID(sqsARN.AccountID)
 		return elasticN.Enable && elasticN.URL != ""
 	} else if isRedisQueue(sqsARN) { // Redis queue.
-		redisN := serverConfig.Notify.GetRedisByID(sqsARN.AccountID)
+		redisN := globalServerConfig.Notify.GetRedisByID(sqsARN.AccountID)
 		return redisN.Enable && redisN.Addr != ""
 	} else if isPostgreSQLQueue(sqsARN) {
-		pgN := serverConfig.Notify.GetPostgreSQLByID(sqsARN.AccountID)
+		pgN := globalServerConfig.Notify.GetPostgreSQLByID(sqsARN.AccountID)
 		// Postgres can work with only default conn. info.
 		return pgN.Enable
 	} else if isMySQLQueue(sqsARN) {
-		msqlN := serverConfig.Notify.GetMySQLByID(sqsARN.AccountID)
+		msqlN := globalServerConfig.Notify.GetMySQLByID(sqsARN.AccountID)
 		// Mysql can work with only default conn. info.
 		return msqlN.Enable
 	} else if isKafkaQueue(sqsARN) {
-		kafkaN := serverConfig.Notify.GetKafkaByID(sqsARN.AccountID)
+		kafkaN := globalServerConfig.Notify.GetKafkaByID(sqsARN.AccountID)
 		return (kafkaN.Enable && len(kafkaN.Brokers) > 0 &&
 			kafkaN.Topic != "")
 	} else if isWebhookQueue(sqsARN) {
-		webhookN := serverConfig.Notify.GetWebhookByID(sqsARN.AccountID)
+		webhookN := globalServerConfig.Notify.GetWebhookByID(sqsARN.AccountID)
 		return webhookN.Enable && webhookN.Endpoint != ""
 	}
 	return false
@@ -273,9 +277,13 @@ func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
 	if len(strs) != 6 {
 		return
 	}
-	if serverConfig.GetRegion() != "" {
+
+	// Server region is allowed to be empty by default,
+	// in such a scenario ARN region is not validating
+	// allowing all regions.
+	if sregion := globalServerConfig.GetRegion(); sregion != "" {
 		region := strs[3]
-		if region != serverConfig.GetRegion() {
+		if region != sregion {
 			return
 		}
 	}

cmd/bucket-policy-handlers.go

@@ -17,7 +17,7 @@
 package cmd
 
 import (
-	"fmt"
+	"encoding/json"
 	"io"
 	"io/ioutil"
 	"net"
@@ -27,7 +27,7 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	mux "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
@@ -36,7 +36,8 @@ const maxAccessPolicySize = 20 * humanize.KiByte
 
 // Verify if a given action is valid for the url path based on the
 // existing bucket access policy.
-func bucketPolicyEvalStatements(action string, resource string, conditions map[string]set.StringSet, statements []policyStatement) bool {
+func bucketPolicyEvalStatements(action string, resource string, conditions policy.ConditionKeyMap,
+	statements []policy.Statement) bool {
 	for _, statement := range statements {
 		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
 			if statement.Effect == "Allow" {
@@ -52,7 +53,8 @@ func bucketPolicyEvalStatements(action string, resource string, conditions map[s
 }
 
 // Verify if action, resource and conditions match input policy statement.
-func bucketPolicyMatchStatement(action string, resource string, conditions map[string]set.StringSet, statement policyStatement) bool {
+func bucketPolicyMatchStatement(action string, resource string, conditions policy.ConditionKeyMap,
+	statement policy.Statement) bool {
 	// Verify if action, resource and condition match in given statement.
 	return (bucketPolicyActionMatch(action, statement) &&
 		bucketPolicyResourceMatch(resource, statement) &&
@@ -60,7 +62,7 @@ func bucketPolicyMatchStatement(action string, resource string, conditions map[s
 }
 
 // Verify if given action matches with policy statement.
-func bucketPolicyActionMatch(action string, statement policyStatement) bool {
+func bucketPolicyActionMatch(action string, statement policy.Statement) bool {
 	return !statement.Actions.FuncMatch(actionMatch, action).IsEmpty()
 }
 
@@ -95,7 +97,7 @@ func isIPInCIDR(cidr, ip string) bool {
 }
 
 // Verify if given resource matches with policy statement.
-func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
+func bucketPolicyResourceMatch(resource string, statement policy.Statement) bool {
 	// the resource rule for object could contain "*" wild card.
 	// the requested object can be given access based on the already set bucket policy if
 	// the match is successful.
@@ -104,7 +106,7 @@ func bucketPolicyResourceMatch(resource string, statement policyStatement) bool
 }
 
 // Verify if given condition matches with policy statement.
-func bucketPolicyConditionMatch(conditions map[string]set.StringSet, statement policyStatement) bool {
+func bucketPolicyConditionMatch(conditions policy.ConditionKeyMap, statement policy.Statement) bool {
 	// Supports following conditions.
 	// - StringEquals
 	// - StringNotEquals
@@ -223,7 +225,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -282,7 +284,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -300,13 +302,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 
 	// Delete bucket access policy, by passing an empty policy
 	// struct.
-	if err := persistAndNotifyBucketPolicyChange(bucket, policyChange{true, nil}, objAPI); err != nil {
-		switch err.(type) {
-		case BucketPolicyNotFound:
-			writeErrorResponse(w, ErrNoSuchBucketPolicy, r.URL)
-		default:
-			writeErrorResponse(w, ErrInternalError, r.URL)
-		}
+	err = persistAndNotifyBucketPolicyChange(bucket, policyChange{
+		true, policy.BucketAccessPolicy{},
+	}, objAPI)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
@@ -325,7 +325,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -345,15 +345,17 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	policy, err := readBucketPolicy(bucket, objAPI)
 	if err != nil {
 		errorIf(err, "Unable to read bucket policy.")
-		switch err.(type) {
-		case BucketPolicyNotFound:
-			writeErrorResponse(w, ErrNoSuchBucketPolicy, r.URL)
-		default:
-			writeErrorResponse(w, ErrInternalError, r.URL)
-		}
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	policyBytes, err := json.Marshal(&policy)
+	if err != nil {
+		errorIf(err, "Unable to marshal bucket policy.")
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Write to client.
-	fmt.Fprint(w, policy)
+	w.Write(policyBytes)
 }

cmd/bucket-policy-handlers_test.go

@@ -25,15 +25,17 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Tests validate Bucket policy resource matcher.
 func TestBucketPolicyResourceMatch(t *testing.T) {
 
 	// generates statement with given resource..
-	generateStatement := func(resource string) policyStatement {
-		statement := policyStatement{}
+	generateStatement := func(resource string) policy.Statement {
+		statement := policy.Statement{}
 		statement.Resources = set.CreateStringSet([]string{resource}...)
 		return statement
 	}
@@ -45,7 +47,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
 
 	testCases := []struct {
 		resourceToMatch       string
-		statement             policyStatement
+		statement             policy.Statement
 		expectedResourceMatch bool
 	}{
 		// Test case 1-4.
@@ -85,7 +87,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
 }
 
 // TestBucketPolicyActionMatch - Test validates whether given action on the
-// bucket/object matches the allowed actions in policyStatement.
+// bucket/object matches the allowed actions in policy.Statement.
 // This test preserves the allowed actions for all 3 sets of policies, that is read-write,read-only, write-only.
 // The intention of the test is to catch any changes made to allowed action for on eof the above 3 major policy groups mentioned.
 func TestBucketPolicyActionMatch(t *testing.T) {
@@ -94,7 +96,7 @@ func TestBucketPolicyActionMatch(t *testing.T) {
 
 	testCases := []struct {
 		action         string
-		statement      policyStatement
+		statement      policy.Statement
 		expectedResult bool
 	}{
 		// s3:GetBucketLocation is the action necessary to be present in the bucket policy to allow
@@ -247,7 +249,7 @@ func TestPutBucketPolicyHandler(t *testing.T) {
 
 // testPutBucketPolicyHandler - Test for Bucket policy end point.
 func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	initBucketPolicies(obj)
 
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
@@ -454,7 +456,7 @@ func TestGetBucketPolicyHandler(t *testing.T) {
 
 // testGetBucketPolicyHandler - Test for end point which fetches the access policy json of the given bucket.
 func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	// initialize bucket policy.
 	initBucketPolicies(obj)
 
@@ -643,7 +645,7 @@ func TestDeleteBucketPolicyHandler(t *testing.T) {
 
 // testDeleteBucketPolicyHandler - Test for Delete bucket policy end point.
 func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	// initialize bucket policy.
 	initBucketPolicies(obj)
 
@@ -843,28 +845,28 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 
 // TestBucketPolicyConditionMatch - Tests to validate whether bucket policy conditions match.
 func TestBucketPolicyConditionMatch(t *testing.T) {
-	// obtain the inner map[string]set.StringSet for policyStatement.Conditions.
+	// obtain the inner map[string]set.StringSet for policy.Statement.Conditions.
 	getInnerMap := func(key2, value string) map[string]set.StringSet {
 		innerMap := make(map[string]set.StringSet)
 		innerMap[key2] = set.CreateStringSet(value)
 		return innerMap
 	}
 
-	// obtain policyStatement with Conditions set.
-	getStatementWithCondition := func(key1, key2, value string) policyStatement {
+	// obtain policy.Statement with Conditions set.
+	getStatementWithCondition := func(key1, key2, value string) policy.Statement {
 		innerMap := getInnerMap(key2, value)
 		// to set policyStatment.Conditions .
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions[key1] = innerMap
 		// new policy statement.
-		statement := policyStatement{}
+		statement := policy.Statement{}
 		// set the condition.
 		statement.Conditions = conditions
 		return statement
 	}
 
 	testCases := []struct {
-		statementCondition policyStatement
+		statementCondition policy.Statement
 		condition          map[string]set.StringSet
 
 		expectedMatch bool

cmd/bucket-policy-parser.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,12 +26,16 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/set"
 )
 
-var conditionKeyActionMap = map[string]set.StringSet{
-	"s3:prefix":   set.CreateStringSet("s3:ListBucket"),
-	"s3:max-keys": set.CreateStringSet("s3:ListBucket"),
+var emptyBucketPolicy = policy.BucketAccessPolicy{}
+
+var conditionKeyActionMap = policy.ConditionKeyMap{
+	"s3:prefix": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads"),
+	"s3:max-keys": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads",
+		"s3:ListMultipartUploadParts"),
 }
 
 // supportedActionMap - lists all the actions supported by minio.
@@ -49,41 +53,16 @@ var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aw
 // supportedEffectMap - supported effects.
 var supportedEffectMap = set.CreateStringSet("Allow", "Deny")
 
-// Statement - minio policy statement
-type policyStatement struct {
-	Actions    set.StringSet                       `json:"Action"`
-	Conditions map[string]map[string]set.StringSet `json:"Condition,omitempty"`
-	Effect     string
-	Principal  interface{}   `json:"Principal"`
-	Resources  set.StringSet `json:"Resource"`
-	Sid        string
-}
-
-// bucketPolicy - collection of various bucket policy statements.
-type bucketPolicy struct {
-	Version    string            // date in YYYY-MM-DD format
-	Statements []policyStatement `json:"Statement"`
-}
-
-// Stringer implementation for the bucket policies.
-func (b bucketPolicy) String() string {
-	bbytes, err := json.Marshal(&b)
-	if err != nil {
-		errorIf(err, "Unable to marshal bucket policy into JSON %#v", b)
-		return ""
-	}
-	return string(bbytes)
-}
-
 // isValidActions - are actions valid.
 func isValidActions(actions set.StringSet) (err error) {
 	// Statement actions cannot be empty.
-	if len(actions) == 0 {
+	if actions.IsEmpty() {
 		err = errors.New("Action list cannot be empty")
 		return err
 	}
 	if unsupportedActions := actions.Difference(supportedActionMap); !unsupportedActions.IsEmpty() {
-		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document", unsupportedActions)
+		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document",
+			unsupportedActions)
 		return err
 	}
 	return nil
@@ -106,7 +85,7 @@ func isValidEffect(effect string) (err error) {
 // isValidResources - are valid resources.
 func isValidResources(resources set.StringSet) (err error) {
 	// Statement resources cannot be empty.
-	if len(resources) == 0 {
+	if resources.IsEmpty() {
 		err = errors.New("Resource list cannot be empty")
 		return err
 	}
@@ -124,60 +103,17 @@ func isValidResources(resources set.StringSet) (err error) {
 	return nil
 }
 
-// Parse principals parses a incoming json. Handles cases for
-// these three combinations.
-// - "Principal": "*",
-// - "Principal": { "AWS" : "*" }
-// - "Principal": { "AWS" : [ "*" ]}
-func parsePrincipals(principal interface{}) set.StringSet {
-	principals, ok := principal.(map[string]interface{})
-	if !ok {
-		var principalStr string
-		principalStr, ok = principal.(string)
-		if ok {
-			return set.CreateStringSet(principalStr)
-		}
-	} // else {
-	var principalStrs []string
-	for _, p := range principals {
-		principalStr, isStr := p.(string)
-		if !isStr {
-			principalsAdd, isInterface := p.([]interface{})
-			if !isInterface {
-				principalStrsAddr, isStrs := p.([]string)
-				if !isStrs {
-					continue
-				}
-				principalStrs = append(principalStrs, principalStrsAddr...)
-			} else {
-				for _, pa := range principalsAdd {
-					var pstr string
-					pstr, isStr = pa.(string)
-					if !isStr {
-						continue
-					}
-					principalStrs = append(principalStrs, pstr)
-				}
-			}
-			continue
-		} // else {
-		principalStrs = append(principalStrs, principalStr)
-	}
-	return set.CreateStringSet(principalStrs...)
-}
-
 // isValidPrincipals - are valid principals.
-func isValidPrincipals(principal interface{}) (err error) {
-	principals := parsePrincipals(principal)
-	// Statement principal should have a value.
-	if len(principals) == 0 {
-		err = errors.New("Principal cannot be empty")
-		return err
+func isValidPrincipals(principal policy.User) (err error) {
+	if principal.AWS.IsEmpty() {
+		return errors.New("Principal cannot be empty")
 	}
-	if unsuppPrincipals := principals.Difference(set.CreateStringSet([]string{"*"}...)); !unsuppPrincipals.IsEmpty() {
+	if diff := principal.AWS.Difference(set.CreateStringSet("*")); !diff.IsEmpty() {
 		// Minio does not support or implement IAM, "*" is the only valid value.
-		// Amazon s3 doc on principals: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
-		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document", unsuppPrincipals)
+		// Amazon s3 doc on principal:
+		// http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
+		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document",
+			diff)
 		return err
 	}
 	return nil
@@ -185,7 +121,7 @@ func isValidPrincipals(principal interface{}) (err error) {
 
 // isValidConditions - returns nil if the given conditions valid and
 // corresponding error otherwise.
-func isValidConditions(actions set.StringSet, conditions map[string]map[string]set.StringSet) (err error) {
+func isValidConditions(actions set.StringSet, conditions policy.ConditionMap) (err error) {
 	// Verify conditions should be valid. Validate if only
 	// supported condition keys are present and return error
 	// otherwise.
@@ -239,7 +175,7 @@ func resourcePrefix(resource string) string {
 // checkBucketPolicyResources validates Resources in unmarshalled bucket policy structure.
 // - Resources are validated against the given set of Actions.
 // -
-func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIErrorCode {
+func checkBucketPolicyResources(bucket string, bucketPolicy policy.BucketAccessPolicy) APIErrorCode {
 	// Validate statements for special actions and collect resources
 	// for others to validate nesting.
 	var resourceMap = set.NewStringSet()
@@ -294,27 +230,27 @@ func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIEr
 
 // parseBucketPolicy - parses and validates if bucket policy is of
 // proper JSON and follows allowed restrictions with policy standards.
-func parseBucketPolicy(bucketPolicyReader io.Reader, policy *bucketPolicy) (err error) {
+func parseBucketPolicy(bucketPolicyReader io.Reader, bktPolicy *policy.BucketAccessPolicy) (err error) {
 	// Parse bucket policy reader.
 	decoder := json.NewDecoder(bucketPolicyReader)
-	if err = decoder.Decode(&policy); err != nil {
+	if err = decoder.Decode(bktPolicy); err != nil {
 		return err
 	}
 
 	// Policy version cannot be empty.
-	if len(policy.Version) == 0 {
+	if len(bktPolicy.Version) == 0 {
 		err = errors.New("Policy version cannot be empty")
 		return err
 	}
 
 	// Policy statements cannot be empty.
-	if len(policy.Statements) == 0 {
+	if len(bktPolicy.Statements) == 0 {
 		err = errors.New("Policy statement cannot be empty")
 		return err
 	}
 
 	// Loop through all policy statements and validate entries.
-	for _, statement := range policy.Statements {
+	for _, statement := range bktPolicy.Statements {
 		// Statement effect should be valid.
 		if err := isValidEffect(statement.Effect); err != nil {
 			return err
@@ -339,19 +275,20 @@ func parseBucketPolicy(bucketPolicyReader io.Reader, policy *bucketPolicy) (err
 
 	// Separate deny and allow statements, so that we can apply deny
 	// statements in the beginning followed by Allow statements.
-	var denyStatements []policyStatement
-	var allowStatements []policyStatement
-	for _, statement := range policy.Statements {
+	var denyStatements []policy.Statement
+	var allowStatements []policy.Statement
+	for _, statement := range bktPolicy.Statements {
 		if statement.Effect == "Deny" {
 			denyStatements = append(denyStatements, statement)
 			continue
 		}
+
 		// else if statement.Effect == "Allow"
 		allowStatements = append(allowStatements, statement)
 	}
 
 	// Deny statements are enforced first once matched.
-	policy.Statements = append(denyStatements, allowStatements...)
+	bktPolicy.Statements = append(denyStatements, allowStatements...)
 
 	// Return successfully parsed policy structure.
 	return nil

cmd/bucket-policy-parser_test.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/minio/minio-go/pkg/policy"
@@ -74,11 +75,11 @@ var (
 )
 
 // Obtain bucket statement for read-write bucketPolicy.
-func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatement {
-	objectResourceStatement := policyStatement{}
+func getReadWriteObjectStatement(bucketName, objectPrefix string) policy.Statement {
+	objectResourceStatement := policy.Statement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	objectResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
 	objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...)
@@ -86,11 +87,11 @@ func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatemen
 }
 
 // Obtain object statement for read-write bucketPolicy.
-func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatement {
-	bucketResourceStatement := policyStatement{}
+func getReadWriteBucketStatement(bucketName, objectPrefix string) policy.Statement {
+	bucketResourceStatement := policy.Statement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	bucketResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
 	bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...)
@@ -98,19 +99,19 @@ func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatemen
 }
 
 // Obtain statements for read-write bucketPolicy.
-func getReadWriteStatement(bucketName, objectPrefix string) []policyStatement {
-	statements := []policyStatement{}
+func getReadWriteStatement(bucketName, objectPrefix string) []policy.Statement {
+	statements := []policy.Statement{}
 	// Save the read write policy.
 	statements = append(statements, getReadWriteBucketStatement(bucketName, objectPrefix), getReadWriteObjectStatement(bucketName, objectPrefix))
 	return statements
 }
 
 // Obtain bucket statement for read only bucketPolicy.
-func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
-	bucketResourceStatement := policyStatement{}
+func getReadOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
+	bucketResourceStatement := policy.Statement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	bucketResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
 	bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...)
@@ -118,11 +119,11 @@ func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement
 }
 
 // Obtain object statement for read only bucketPolicy.
-func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
-	objectResourceStatement := policyStatement{}
+func getReadOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
+	objectResourceStatement := policy.Statement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	objectResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
 	objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...)
@@ -130,20 +131,20 @@ func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement
 }
 
 // Obtain statements for read only bucketPolicy.
-func getReadOnlyStatement(bucketName, objectPrefix string) []policyStatement {
-	statements := []policyStatement{}
+func getReadOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
+	statements := []policy.Statement{}
 	// Save the read only policy.
 	statements = append(statements, getReadOnlyBucketStatement(bucketName, objectPrefix), getReadOnlyObjectStatement(bucketName, objectPrefix))
 	return statements
 }
 
 // Obtain bucket statements for write only bucketPolicy.
-func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
+func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
 
-	bucketResourceStatement := policyStatement{}
+	bucketResourceStatement := policy.Statement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	bucketResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
 	bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...)
@@ -151,11 +152,11 @@ func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatemen
 }
 
 // Obtain object statements for write only bucketPolicy.
-func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
-	objectResourceStatement := policyStatement{}
+func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
+	objectResourceStatement := policy.Statement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	objectResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
 	objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...)
@@ -163,8 +164,8 @@ func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatemen
 }
 
 // Obtain statements for write only bucketPolicy.
-func getWriteOnlyStatement(bucketName, objectPrefix string) []policyStatement {
-	statements := []policyStatement{}
+func getWriteOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
+	statements := []policy.Statement{}
 	// Write only policy.
 	// Save the write only policy.
 	statements = append(statements, getWriteOnlyBucketStatement(bucketName, objectPrefix), getWriteOnlyBucketStatement(bucketName, objectPrefix))
@@ -325,9 +326,10 @@ func TestIsValidPrincipals(t *testing.T) {
 		{[]string{"*"}, nil, true},
 	}
 	for i, testCase := range testCases {
-		err := isValidPrincipals(map[string]interface{}{
-			"AWS": testCase.principals,
-		})
+		u := policy.User{
+			AWS: set.CreateStringSet(testCase.principals...),
+		}
+		err := isValidPrincipals(u)
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
 		}
@@ -343,86 +345,86 @@ func TestIsValidPrincipals(t *testing.T) {
 	}
 }
 
-// getEmptyConditionKeyMap - returns a function that generates a
+// getEmptyConditionMap - returns a function that generates a
 // condition key map for a given key.
-func getEmptyConditionKeyMap(conditionKey string) func() map[string]map[string]set.StringSet {
-	emptyConditonGenerator := func() map[string]map[string]set.StringSet {
-		emptyMap := make(map[string]set.StringSet)
-		conditions := make(map[string]map[string]set.StringSet)
+func getEmptyConditionMap(conditionKey string) func() policy.ConditionMap {
+	emptyConditonGenerator := func() policy.ConditionMap {
+		emptyMap := make(policy.ConditionKeyMap)
+		conditions := make(policy.ConditionMap)
 		conditions[conditionKey] = emptyMap
 		return conditions
 	}
 	return emptyConditonGenerator
 }
 
-// Tests validate policyStatement condition validator.
+// Tests validate policy.Statement condition validator.
 func TestIsValidConditions(t *testing.T) {
 	// returns empty conditions map.
-	setEmptyConditions := func() map[string]map[string]set.StringSet {
-		return make(map[string]map[string]set.StringSet)
+	setEmptyConditions := func() policy.ConditionMap {
+		return make(policy.ConditionMap)
 	}
 
 	// returns map with the "StringEquals" set to empty map.
-	setEmptyStringEquals := getEmptyConditionKeyMap("StringEquals")
+	setEmptyStringEquals := getEmptyConditionMap("StringEquals")
 
 	// returns map with the "StringNotEquals" set to empty map.
-	setEmptyStringNotEquals := getEmptyConditionKeyMap("StringNotEquals")
+	setEmptyStringNotEquals := getEmptyConditionMap("StringNotEquals")
 
 	// returns map with the "StringLike" set to empty map.
-	setEmptyStringLike := getEmptyConditionKeyMap("StringLike")
+	setEmptyStringLike := getEmptyConditionMap("StringLike")
 
 	// returns map with the "StringNotLike" set to empty map.
-	setEmptyStringNotLike := getEmptyConditionKeyMap("StringNotLike")
+	setEmptyStringNotLike := getEmptyConditionMap("StringNotLike")
 
 	// returns map with the "IpAddress" set to empty map.
-	setEmptyIPAddress := getEmptyConditionKeyMap("IpAddress")
+	setEmptyIPAddress := getEmptyConditionMap("IpAddress")
 
 	// returns map with "NotIpAddress" set to empty map.
-	setEmptyNotIPAddress := getEmptyConditionKeyMap("NotIpAddress")
+	setEmptyNotIPAddress := getEmptyConditionMap("NotIpAddress")
 
 	// Generate conditions.
-	generateConditions := func(key1, key2, value string) map[string]map[string]set.StringSet {
-		innerMap := make(map[string]set.StringSet)
+	generateConditions := func(key1, key2, value string) policy.ConditionMap {
+		innerMap := make(policy.ConditionKeyMap)
 		innerMap[key2] = set.CreateStringSet(value)
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions[key1] = innerMap
 		return conditions
 	}
 
 	// generate ambigious conditions.
-	generateAmbigiousConditions := func() map[string]map[string]set.StringSet {
-		prefixMap := make(map[string]set.StringSet)
+	generateAmbigiousConditions := func() policy.ConditionMap {
+		prefixMap := make(policy.ConditionKeyMap)
 		prefixMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions["StringEquals"] = prefixMap
 		conditions["StringNotEquals"] = prefixMap
 		return conditions
 	}
 
 	// generate valid and non valid type in the condition map.
-	generateValidInvalidConditions := func() map[string]map[string]set.StringSet {
-		innerMap := make(map[string]set.StringSet)
+	generateValidInvalidConditions := func() policy.ConditionMap {
+		innerMap := make(policy.ConditionKeyMap)
 		innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions["StringEquals"] = innerMap
 		conditions["InvalidType"] = innerMap
 		return conditions
 	}
 
 	// generate valid and invalid keys for valid types in the same condition map.
-	generateValidInvalidConditionKeys := func() map[string]map[string]set.StringSet {
-		innerMapValid := make(map[string]set.StringSet)
+	generateValidInvalidConditionKeys := func() policy.ConditionMap {
+		innerMapValid := make(policy.ConditionKeyMap)
 		innerMapValid["s3:prefix"] = set.CreateStringSet("Asia/")
 		innerMapInValid := make(map[string]set.StringSet)
 		innerMapInValid["s3:invalid"] = set.CreateStringSet("Asia/")
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions["StringEquals"] = innerMapValid
 		conditions["StringEquals"] = innerMapInValid
 		return conditions
 	}
 
 	// List of Conditions used for test cases.
-	testConditions := []map[string]map[string]set.StringSet{
+	testConditions := []policy.ConditionMap{
 		generateConditions("StringValues", "s3:max-keys", "100"),
 		generateConditions("StringEquals", "s3:Object", "100"),
 		generateAmbigiousConditions(),
@@ -447,7 +449,7 @@ func TestIsValidConditions(t *testing.T) {
 		"please validate your policy document", "s3:max-keys", getObjectActionSet)
 	testCases := []struct {
 		inputActions   set.StringSet
-		inputCondition map[string]map[string]set.StringSet
+		inputCondition policy.ConditionMap
 		// expected result.
 		expectedErr error
 		// flag indicating whether test should pass.
@@ -518,26 +520,26 @@ func TestIsValidConditions(t *testing.T) {
 // Tests validate Policy Action and Resource fields.
 func TestCheckbucketPolicyResources(t *testing.T) {
 	// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
-	setValidPrefixActions := func(statements []policyStatement) []policyStatement {
+	setValidPrefixActions := func(statements []policy.Statement) []policy.Statement {
 		statements[0].Actions = set.CreateStringSet([]string{"s3:DeleteObject", "s3:PutObject"}...)
 		return statements
 	}
 	// contracting policy statement with recursive resources.
 	// should result in ErrMalformedPolicy
-	setRecurseResource := func(statements []policyStatement) []policyStatement {
+	setRecurseResource := func(statements []policy.Statement) []policy.Statement {
 		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}...)
 		return statements
 	}
 
 	// constructing policy statement with lexically close characters.
 	// should not result in ErrMalformedPolicy
-	setResourceLexical := func(statements []policyStatement) []policyStatement {
+	setResourceLexical := func(statements []policy.Statement) []policy.Statement {
 		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}...)
 		return statements
 	}
 
 	// List of bucketPolicy used for tests.
-	bucketAccessPolicies := []bucketPolicy{
+	bucketAccessPolicies := []policy.BucketAccessPolicy{
 		// bucketPolicy - 1.
 		// Contains valid read only policy statement.
 		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
@@ -568,7 +570,7 @@ func TestCheckbucketPolicyResources(t *testing.T) {
 	}
 
 	testCases := []struct {
-		inputPolicy bucketPolicy
+		inputPolicy policy.BucketAccessPolicy
 		// expected results.
 		apiErrCode APIErrorCode
 		// Flag indicating whether the test should pass.
@@ -599,7 +601,7 @@ func TestCheckbucketPolicyResources(t *testing.T) {
 		{bucketAccessPolicies[6], ErrNone, true},
 	}
 	for i, testCase := range testCases {
-		apiErrCode := checkBucketPolicyResources("minio-bucket", &testCase.inputPolicy)
+		apiErrCode := checkBucketPolicyResources("minio-bucket", testCase.inputPolicy)
 		if apiErrCode != ErrNone && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with Errocode %v", i+1, apiErrCode)
 		}
@@ -618,39 +620,39 @@ func TestCheckbucketPolicyResources(t *testing.T) {
 // Tests validate parsing of BucketAccessPolicy.
 func TestParseBucketPolicy(t *testing.T) {
 	// set Unsupported Actions.
-	setUnsupportedActions := func(statements []policyStatement) []policyStatement {
+	setUnsupportedActions := func(statements []policy.Statement) []policy.Statement {
 		// "s3:DeleteEverything"" is an Unsupported Action.
 		statements[0].Actions = set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...)
 		return statements
 	}
 	// set unsupported Effect.
-	setUnsupportedEffect := func(statements []policyStatement) []policyStatement {
+	setUnsupportedEffect := func(statements []policy.Statement) []policy.Statement {
 		// Effect "Don't allow" is Unsupported.
 		statements[0].Effect = "DontAllow"
 		return statements
 	}
 	// set unsupported principals.
-	setUnsupportedPrincipals := func(statements []policyStatement) []policyStatement {
+	setUnsupportedPrincipals := func(statements []policy.Statement) []policy.Statement {
 		// "User1111"" is an Unsupported Principal.
-		statements[0].Principal = map[string]interface{}{
-			"AWS": []string{"*", "User1111"},
+		statements[0].Principal = policy.User{
+			AWS: set.CreateStringSet([]string{"*", "User1111"}...),
 		}
 		return statements
 	}
 	// set unsupported Resources.
-	setUnsupportedResources := func(statements []policyStatement) []policyStatement {
+	setUnsupportedResources := func(statements []policy.Statement) []policy.Statement {
 		// "s3:DeleteEverything"" is an Unsupported Action.
 		statements[0].Resources = set.CreateStringSet([]string{"my-resource"}...)
 		return statements
 	}
 	// List of bucketPolicy used for test cases.
-	bucketAccesPolicies := []bucketPolicy{
+	bucketAccesPolicies := []policy.BucketAccessPolicy{
 		// bucketPolicy - 0.
 		// bucketPolicy statement empty.
 		{Version: "1.0"},
 		// bucketPolicy - 1.
 		// bucketPolicy version empty.
-		{Version: "", Statements: []policyStatement{}},
+		{Version: "", Statements: []policy.Statement{}},
 		// bucketPolicy - 2.
 		// Readonly bucketPolicy.
 		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
@@ -675,19 +677,19 @@ func TestParseBucketPolicy(t *testing.T) {
 	}
 
 	testCases := []struct {
-		inputPolicy bucketPolicy
+		inputPolicy policy.BucketAccessPolicy
 		// expected results.
-		expectedPolicy bucketPolicy
+		expectedPolicy policy.BucketAccessPolicy
 		err            error
 		// Flag indicating whether the test should pass.
 		shouldPass bool
 	}{
 		// Test case - 1.
 		// bucketPolicy statement empty.
-		{bucketAccesPolicies[0], bucketPolicy{}, errors.New("Policy statement cannot be empty"), false},
+		{bucketAccesPolicies[0], policy.BucketAccessPolicy{}, errors.New("Policy statement cannot be empty"), false},
 		// Test case - 2.
 		// bucketPolicy version empty.
-		{bucketAccesPolicies[1], bucketPolicy{}, errors.New("Policy version cannot be empty"), false},
+		{bucketAccesPolicies[1], policy.BucketAccessPolicy{}, errors.New("Policy version cannot be empty"), false},
 		// Test case - 3.
 		// Readonly bucketPolicy.
 		{bucketAccesPolicies[2], bucketAccesPolicies[2], nil, true},
@@ -718,8 +720,8 @@ func TestParseBucketPolicy(t *testing.T) {
 			t.Fatalf("Test %d: Couldn't Marshal bucket policy %s", i+1, err)
 		}
 
-		var actualAccessPolicy = &bucketPolicy{}
-		err = parseBucketPolicy(&buffer, actualAccessPolicy)
+		var actualAccessPolicy = policy.BucketAccessPolicy{}
+		err = parseBucketPolicy(&buffer, &actualAccessPolicy)
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
 		}
@@ -734,7 +736,7 @@ func TestParseBucketPolicy(t *testing.T) {
 		}
 		// Test passes as expected, but the output values are verified for correctness here.
 		if err == nil && testCase.shouldPass {
-			if testCase.expectedPolicy.String() != actualAccessPolicy.String() {
+			if !reflect.DeepEqual(testCase.expectedPolicy, actualAccessPolicy) {
 				t.Errorf("Test %d: The expected statements from resource statement generator doesn't match the actual statements", i+1)
 			}
 		}
@@ -751,8 +753,8 @@ func TestAWSRefererCondition(t *testing.T) {
 		set.CreateStringSet("www.example.com",
 			"http://www.example.com"))
 
-	requestConditionKeyMap := make(map[string]set.StringSet)
-	requestConditionKeyMap["referer"] = set.CreateStringSet("www.example.com")
+	requestConditionMap := make(policy.ConditionKeyMap)
+	requestConditionMap["referer"] = set.CreateStringSet("www.example.com")
 
 	testCases := []struct {
 		effect       string
@@ -782,20 +784,20 @@ func TestAWSRefererCondition(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions[test.conditionKey] = conditionsKeyMap
 
-		allowStatement := policyStatement{
+		allowStatement := policy.Statement{
 			Sid:    "Testing AWS referer condition",
 			Effect: test.effect,
-			Principal: map[string]interface{}{
-				"AWS": "*",
+			Principal: policy.User{
+				AWS: set.CreateStringSet("*"),
 			},
 			Resources:  resource,
 			Conditions: conditions,
 		}
 
-		if result := bucketPolicyConditionMatch(requestConditionKeyMap, allowStatement); result != test.match {
+		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
 			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
 				i+1, test.match, result)
 		}
@@ -813,8 +815,8 @@ func TestAWSSourceIPCondition(t *testing.T) {
 		set.CreateStringSet("54.240.143.0/24",
 			"2001:DB8:1234:5678::/64"))
 
-	requestConditionKeyMap := make(map[string]set.StringSet)
-	requestConditionKeyMap["ip"] = set.CreateStringSet("54.240.143.2")
+	requestConditionMap := make(policy.ConditionKeyMap)
+	requestConditionMap["ip"] = set.CreateStringSet("54.240.143.2")
 
 	testCases := []struct {
 		effect       string
@@ -844,20 +846,20 @@ func TestAWSSourceIPCondition(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions[test.conditionKey] = conditionsKeyMap
 
-		allowStatement := policyStatement{
+		allowStatement := policy.Statement{
 			Sid:    "Testing AWS referer condition",
 			Effect: test.effect,
-			Principal: map[string]interface{}{
-				"AWS": "*",
+			Principal: policy.User{
+				AWS: set.CreateStringSet("*"),
 			},
 			Resources:  resource,
 			Conditions: conditions,
 		}
 
-		if result := bucketPolicyConditionMatch(requestConditionKeyMap, allowStatement); result != test.match {
+		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
 			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
 				i+1, test.match, result)
 		}

cmd/bucket-policy.go

@@ -20,8 +20,11 @@ import (
 	"bytes"
 	"encoding/json"
 	"io"
+	"reflect"
 	"sync"
 
+	"github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
 )
 
@@ -43,7 +46,7 @@ type bucketPolicies struct {
 	rwMutex *sync.RWMutex
 
 	// Collection of 'bucket' policies.
-	bucketPolicyConfigs map[string]*bucketPolicy
+	bucketPolicyConfigs map[string]policy.BucketAccessPolicy
 }
 
 // Represent a policy change
@@ -53,11 +56,11 @@ type policyChange struct {
 	IsRemove bool
 
 	// represents the new policy for the bucket
-	BktPolicy *bucketPolicy
+	BktPolicy policy.BucketAccessPolicy
 }
 
 // Fetch bucket policy for a given bucket.
-func (bp bucketPolicies) GetBucketPolicy(bucket string) *bucketPolicy {
+func (bp bucketPolicies) GetBucketPolicy(bucket string) policy.BucketAccessPolicy {
 	bp.rwMutex.RLock()
 	defer bp.rwMutex.RUnlock()
 	return bp.bucketPolicyConfigs[bucket]
@@ -72,7 +75,7 @@ func (bp *bucketPolicies) SetBucketPolicy(bucket string, pCh policyChange) error
 	if pCh.IsRemove {
 		delete(bp.bucketPolicyConfigs, bucket)
 	} else {
-		if pCh.BktPolicy == nil {
+		if reflect.DeepEqual(pCh.BktPolicy, emptyBucketPolicy) {
 			return errInvalidArgument
 		}
 		bp.bucketPolicyConfigs[bucket] = pCh.BktPolicy
@@ -81,23 +84,23 @@ func (bp *bucketPolicies) SetBucketPolicy(bucket string, pCh policyChange) error
 }
 
 // Loads all bucket policies from persistent layer.
-func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]*bucketPolicy, err error) {
+func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]policy.BucketAccessPolicy, err error) {
 	// List buckets to proceed loading all notification configuration.
 	buckets, err := objAPI.ListBuckets()
-	errorIf(err, "Unable to list buckets.")
 	if err != nil {
-		return nil, errorCause(err)
+		errorIf(err, "Unable to list buckets.")
+		return nil, errors.Cause(err)
 	}
 
-	policies = make(map[string]*bucketPolicy)
+	policies = make(map[string]policy.BucketAccessPolicy)
 	var pErrs []error
 	// Loads bucket policy.
 	for _, bucket := range buckets {
-		policy, pErr := readBucketPolicy(bucket.Name, objAPI)
+		bp, pErr := readBucketPolicy(bucket.Name, objAPI)
 		if pErr != nil {
 			// net.Dial fails for rpc client or any
 			// other unexpected errors during net.Dial.
-			if !isErrIgnored(pErr, errDiskNotFound) {
+			if !errors.IsErrIgnored(pErr, errDiskNotFound) {
 				if !isErrBucketPolicyNotFound(pErr) {
 					pErrs = append(pErrs, pErr)
 				}
@@ -105,7 +108,7 @@ func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]*bucketPolic
 			// Continue to load other bucket policies if possible.
 			continue
 		}
-		policies[bucket.Name] = policy
+		policies[bucket.Name] = bp
 	}
 
 	// Look for any errors occurred while reading bucket policies.
@@ -160,7 +163,7 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader
 			return nil, BucketPolicyNotFound{Bucket: bucket}
 		}
 		errorIf(err, "Unable to load policy for the bucket %s.", bucket)
-		return nil, errorCause(err)
+		return nil, errors.Cause(err)
 	}
 
 	return &buffer, nil
@@ -168,21 +171,20 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader
 
 // readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
 // if bucket policy is not found. This function also parses the bucket policy into an object.
-func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error) {
+func readBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPolicy, error) {
 	// Read bucket policy JSON.
 	bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI)
 	if err != nil {
-		return nil, err
+		return emptyBucketPolicy, err
 	}
 
 	// Parse the saved policy.
-	var policy = &bucketPolicy{}
-	err = parseBucketPolicy(bucketPolicyReader, policy)
-	if err != nil {
-		return nil, err
+	var bp policy.BucketAccessPolicy
+	if err = parseBucketPolicy(bucketPolicyReader, &bp); err != nil {
+		return emptyBucketPolicy, err
 
 	}
-	return policy, nil
+	return bp, nil
 }
 
 // removeBucketPolicy - removes any previously written bucket policy. Returns BucketPolicyNotFound
@@ -195,9 +197,10 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
 		return err
 	}
 	defer objLock.Unlock()
-	if err := objAPI.DeleteObject(minioMetaBucket, policyPath); err != nil {
+	err := objAPI.DeleteObject(minioMetaBucket, policyPath)
+	if err != nil {
 		errorIf(err, "Unable to remove bucket-policy on bucket %s.", bucket)
-		err = errorCause(err)
+		err = errors.Cause(err)
 		if _, ok := err.(ObjectNotFound); ok {
 			return BucketPolicyNotFound{Bucket: bucket}
 		}
@@ -207,10 +210,10 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
 }
 
 // writeBucketPolicy - save a bucket policy that is assumed to be validated.
-func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy *bucketPolicy) error {
+func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy policy.BucketAccessPolicy) error {
 	buf, err := json.Marshal(bpy)
 	if err != nil {
-		errorIf(err, "Unable to marshal bucket policy '%v' to JSON", *bpy)
+		errorIf(err, "Unable to marshal bucket policy '%#v' to JSON", bpy)
 		return err
 	}
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
@@ -224,27 +227,27 @@ func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy *bucketPolicy) err
 	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
 	if err != nil {
 		errorIf(err, "Unable to set policy for the bucket %s", bucket)
-		return errorCause(err)
+		return errors.Cause(err)
 	}
 
 	if _, err = objAPI.PutObject(minioMetaBucket, policyPath, hashReader, nil); err != nil {
 		errorIf(err, "Unable to set policy for the bucket %s", bucket)
-		return errorCause(err)
+		return errors.Cause(err)
 	}
 	return nil
 }
 
 func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI ObjectLayer) APIErrorCode {
 	// Parse bucket policy.
-	var policy = &bucketPolicy{}
-	err := parseBucketPolicy(bytes.NewReader(policyBytes), policy)
+	var bktPolicy policy.BucketAccessPolicy
+	err := parseBucketPolicy(bytes.NewReader(policyBytes), &bktPolicy)
 	if err != nil {
 		errorIf(err, "Unable to parse bucket policy.")
 		return ErrInvalidPolicyDocument
 	}
 
 	// Parse check bucket policy.
-	if s3Error := checkBucketPolicyResources(bucket, policy); s3Error != ErrNone {
+	if s3Error := checkBucketPolicyResources(bucket, bktPolicy); s3Error != ErrNone {
 		return s3Error
 	}
 
@@ -257,7 +260,7 @@ func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI Objec
 	defer bucketLock.Unlock()
 
 	// Save bucket policy.
-	if err = persistAndNotifyBucketPolicyChange(bucket, policyChange{false, policy}, objAPI); err != nil {
+	if err = persistAndNotifyBucketPolicyChange(bucket, policyChange{false, bktPolicy}, objAPI); err != nil {
 		switch err.(type) {
 		case BucketNameInvalid:
 			return ErrInvalidBucketName
@@ -276,11 +279,12 @@ func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI Objec
 // change. In-memory state is updated in response to the notification.
 func persistAndNotifyBucketPolicyChange(bucket string, pCh policyChange, objAPI ObjectLayer) error {
 	if pCh.IsRemove {
-		if err := removeBucketPolicy(bucket, objAPI); err != nil {
+		err := removeBucketPolicy(bucket, objAPI)
+		if err != nil {
 			return err
 		}
 	} else {
-		if pCh.BktPolicy == nil {
+		if reflect.DeepEqual(pCh.BktPolicy, emptyBucketPolicy) {
 			return errInvalidArgument
 		}
 		if err := writeBucketPolicy(bucket, objAPI, pCh.BktPolicy); err != nil {

cmd/build-constants.go

@@ -16,11 +16,13 @@
 
 package cmd
 
+import "go/build"
+
 // DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
 // set through ‘buildscripts/gen-ldflags.go’.
 var (
 	// GOPATH - GOPATH value at the time of build.
-	GOPATH = ""
+	GOPATH = build.Default.GOPATH
 
 	// Go get development tag.
 	goGetTag = "DEVELOPMENT.GOGET"

cmd/common-main.go

@@ -24,39 +24,26 @@ import (
 	"time"
 
 	"github.com/minio/cli"
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Check for updates and print a notification message
 func checkUpdate(mode string) {
-	// Its OK to ignore any errors during getUpdateInfo() here.
-	if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil {
-		if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" {
+	// Its OK to ignore any errors during doUpdate() here.
+	if updateMsg, _, currentReleaseTime, latestReleaseTime, err := getUpdateInfo(2*time.Second, mode); err == nil {
+		if globalInplaceUpdateDisabled {
 			log.Println(updateMsg)
+		} else {
+			log.Println(prepareUpdateMessage("Run `minio update`", latestReleaseTime.Sub(currentReleaseTime)))
 		}
 	}
 }
 
-func enableLoggers() {
-	fileLogTarget := serverConfig.Logger.GetFile()
-	if fileLogTarget.Enable {
-		err := InitFileLogger(&fileLogTarget)
-		fatalIf(err, "Unable to initialize file logger")
-		log.AddTarget(fileLogTarget)
-	}
-
-	consoleLogTarget := serverConfig.Logger.GetConsole()
-	if consoleLogTarget.Enable {
-		InitConsoleLogger(&consoleLogTarget)
-	}
-
-	log.SetConsoleTarget(consoleLogTarget)
-}
-
 func initConfig() {
 	// Config file does not exist, we create it fresh and return upon success.
 	if isFile(getConfigFile()) {
 		fatalIf(migrateConfig(), "Config migration failed.")
-		fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19)
+		fatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
 	} else {
 		fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
 		log.Println("Created minio configuration file successfully at " + getConfigDir())
@@ -95,7 +82,7 @@ func handleCommonEnvVars() {
 	accessKey := os.Getenv("MINIO_ACCESS_KEY")
 	secretKey := os.Getenv("MINIO_SECRET_KEY")
 	if accessKey != "" && secretKey != "" {
-		cred, err := createCredential(accessKey, secretKey)
+		cred, err := auth.CreateCredentials(accessKey, secretKey)
 		fatalIf(err, "Invalid access/secret Key set in environment.")
 
 		// credential Envs are set globally.
@@ -116,4 +103,44 @@ func handleCommonEnvVars() {
 	}
 
 	globalHTTPTrace = os.Getenv("MINIO_HTTP_TRACE") != ""
+
+	globalDomainName = os.Getenv("MINIO_DOMAIN")
+	if globalDomainName != "" {
+		globalIsEnvDomainName = true
+	}
+
+	// In place update is true by default if the MINIO_UPDATE is not set
+	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
+	// in-place update is off.
+	globalInplaceUpdateDisabled = strings.EqualFold(os.Getenv("MINIO_UPDATE"), "off")
+
+	// Validate and store the storage class env variables only for XL/Dist XL setups
+	if globalIsXL {
+		var err error
+
+		// Check for environment variables and parse into storageClass struct
+		if ssc := os.Getenv(standardStorageClassEnv); ssc != "" {
+			globalStandardStorageClass, err = parseStorageClass(ssc)
+			fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
+		}
+
+		if rrsc := os.Getenv(reducedRedundancyStorageClassEnv); rrsc != "" {
+			globalRRStorageClass, err = parseStorageClass(rrsc)
+			fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
+		}
+
+		// Validation is done after parsing both the storage classes. This is needed because we need one
+		// storage class value to deduce the correct value of the other storage class.
+		if globalRRStorageClass.Scheme != "" {
+			err := validateRRSParity(globalRRStorageClass.Parity, globalStandardStorageClass.Parity)
+			fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
+			globalIsStorageClass = true
+		}
+
+		if globalStandardStorageClass.Scheme != "" {
+			err := validateSSParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
+			fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
+			globalIsStorageClass = true
+		}
+	}
 }

cmd/config-current.go

@@ -20,41 +20,35 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"strconv"
 	"sync"
 
+	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/quick"
 	"github.com/tidwall/gjson"
 )
 
+// Steps to move from version N to version N+1
+// 1. Add new struct serverConfigVN+1 in config-versions.go
+// 2. Set serverConfigVersion to "N+1"
+// 3. Set serverConfig to serverConfigVN+1
+// 4. Add new migration function (ex. func migrateVNToVN+1()) in config-migrate.go
+// 5. Call migrateVNToVN+1() from migrateConfig() in config-migrate.go
+// 6. Make changes in config-current_test.go for any test change
+
 // Config version
-const v19 = "19"
+const serverConfigVersion = "22"
+
+type serverConfig = serverConfigV22
 
 var (
-	// serverConfig server config.
-	serverConfig   *serverConfigV19
-	serverConfigMu sync.RWMutex
+	// globalServerConfig server config.
+	globalServerConfig   *serverConfig
+	globalServerConfigMu sync.RWMutex
 )
 
-// serverConfigV19 server configuration version '19' which is like
-// version '18' except it adds support for MQTT notifications.
-type serverConfigV19 struct {
-	sync.RWMutex
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential credential  `json:"credential"`
-	Region     string      `json:"region"`
-	Browser    BrowserFlag `json:"browser"`
-
-	// Additional error logging configuration.
-	Logger *loggers `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
-}
-
 // GetVersion get current config version.
-func (s *serverConfigV19) GetVersion() string {
+func (s *serverConfig) GetVersion() string {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -62,7 +56,7 @@ func (s *serverConfigV19) GetVersion() string {
 }
 
 // SetRegion set a new region.
-func (s *serverConfigV19) SetRegion(region string) {
+func (s *serverConfig) SetRegion(region string) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -71,15 +65,15 @@ func (s *serverConfigV19) SetRegion(region string) {
 }
 
 // GetRegion get current region.
-func (s *serverConfigV19) GetRegion() string {
+func (s *serverConfig) GetRegion() string {
 	s.RLock()
 	defer s.RUnlock()
 
 	return s.Region
 }
 
-// SetCredentials set new credentials. SetCredential returns the previous credential.
-func (s *serverConfigV19) SetCredential(creds credential) (prevCred credential) {
+// SetCredential sets new credential and returns the previous credential.
+func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -94,7 +88,7 @@ func (s *serverConfigV19) SetCredential(creds credential) (prevCred credential)
 }
 
 // GetCredentials get current credentials.
-func (s *serverConfigV19) GetCredential() credential {
+func (s *serverConfig) GetCredential() auth.Credentials {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -102,7 +96,7 @@ func (s *serverConfigV19) GetCredential() credential {
 }
 
 // SetBrowser set if browser is enabled.
-func (s *serverConfigV19) SetBrowser(b bool) {
+func (s *serverConfig) SetBrowser(b bool) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -110,8 +104,54 @@ func (s *serverConfigV19) SetBrowser(b bool) {
 	s.Browser = BrowserFlag(b)
 }
 
+func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
+	s.Lock()
+	defer s.Unlock()
+
+	// Set the values
+	s.StorageClass.Standard = standardClass.Scheme + strconv.Itoa(standardClass.Parity)
+	s.StorageClass.RRS = rrsClass.Scheme + strconv.Itoa(rrsClass.Parity)
+}
+
+// GetStorageClass reads storage class fields from current config, parses and validates it.
+// It returns the standard and reduced redundancy storage class struct
+func (s *serverConfig) GetStorageClass() (ssc, rrsc storageClass) {
+	s.RLock()
+	defer s.RUnlock()
+
+	var err error
+
+	if s.StorageClass.Standard != "" {
+		// Parse the values read from config file into storageClass struct
+		ssc, err = parseStorageClass(s.StorageClass.Standard)
+		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.Standard)
+	}
+
+	if s.StorageClass.RRS != "" {
+		// Parse the values read from config file into storageClass struct
+		rrsc, err = parseStorageClass(s.StorageClass.RRS)
+		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.RRS)
+	}
+
+	// Validation is done after parsing both the storage classes. This is needed because we need one
+	// storage class value to deduce the correct value of the other storage class.
+	if rrsc.Scheme != "" {
+		err = validateRRSParity(rrsc.Parity, ssc.Parity)
+		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.RRS)
+		globalIsStorageClass = true
+	}
+
+	if ssc.Scheme != "" {
+		err = validateSSParity(ssc.Parity, rrsc.Parity)
+		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.Standard)
+		globalIsStorageClass = true
+	}
+
+	return
+}
+
 // GetCredentials get current credentials.
-func (s *serverConfigV19) GetBrowser() bool {
+func (s *serverConfig) GetBrowser() bool {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -119,7 +159,7 @@ func (s *serverConfigV19) GetBrowser() bool {
 }
 
 // Save config.
-func (s *serverConfigV19) Save() error {
+func (s *serverConfig) Save() error {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -127,19 +167,15 @@ func (s *serverConfigV19) Save() error {
 	return quick.Save(getConfigFile(), s)
 }
 
-func newServerConfigV19() *serverConfigV19 {
-	srvCfg := &serverConfigV19{
-		Version:    v19,
-		Credential: mustGetNewCredential(),
+func newServerConfig() *serverConfig {
+	srvCfg := &serverConfig{
+		Version:    serverConfigVersion,
+		Credential: auth.MustGetNewCredentials(),
 		Region:     globalMinioDefaultRegion,
 		Browser:    true,
-		Logger:     &loggers{},
 		Notify:     &notifier{},
 	}
 
-	// Enable console logger by default on a fresh run.
-	srvCfg.Logger.Console = NewConsoleLogger()
-
 	// Make sure to initialize notification configs.
 	srvCfg.Notify.AMQP = make(map[string]amqpNotify)
 	srvCfg.Notify.AMQP["1"] = amqpNotify{}
@@ -167,7 +203,7 @@ func newServerConfigV19() *serverConfigV19 {
 // found, otherwise use default parameters
 func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfigV19()
+	srvCfg := newServerConfig()
 
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
@@ -182,15 +218,23 @@ func newConfig() error {
 		srvCfg.SetRegion(globalServerRegion)
 	}
 
+	if globalIsEnvDomainName {
+		srvCfg.Domain = globalDomainName
+	}
+
+	if globalIsStorageClass {
+		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
+	}
+
 	// hold the mutex lock before a new config is assigned.
 	// Save the new config globally.
 	// unlock the mutex.
-	serverConfigMu.Lock()
-	serverConfig = srvCfg
-	serverConfigMu.Unlock()
+	globalServerConfigMu.Lock()
+	globalServerConfig = srvCfg
+	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return serverConfig.Save()
+	return globalServerConfig.Save()
 }
 
 // doCheckDupJSONKeys recursively detects duplicate json keys
@@ -245,8 +289,8 @@ func checkDupJSONKeys(json string) error {
 }
 
 // getValidConfig - returns valid server configuration
-func getValidConfig() (*serverConfigV19, error) {
-	srvCfg := &serverConfigV19{
+func getValidConfig() (*serverConfig, error) {
+	srvCfg := &serverConfig{
 		Region:  globalMinioDefaultRegion,
 		Browser: true,
 	}
@@ -256,8 +300,8 @@ func getValidConfig() (*serverConfigV19, error) {
 		return nil, err
 	}
 
-	if srvCfg.Version != v19 {
-		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v19, srvCfg.Version)
+	if srvCfg.Version != serverConfigVersion {
+		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
 	}
 
 	// Load config file json and check for duplication json keys
@@ -277,11 +321,6 @@ func getValidConfig() (*serverConfigV19, error) {
 		return nil, errors.New("invalid credential in config file " + configFile)
 	}
 
-	// Validate logger field
-	if err = srvCfg.Logger.Validate(); err != nil {
-		return nil, err
-	}
-
 	// Validate notify field
 	if err = srvCfg.Notify.Validate(); err != nil {
 		return nil, err
@@ -311,19 +350,33 @@ func loadConfig() error {
 		srvCfg.SetRegion(globalServerRegion)
 	}
 
+	if globalIsEnvDomainName {
+		srvCfg.Domain = globalDomainName
+	}
+
+	if globalIsStorageClass {
+		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
+	}
+
 	// hold the mutex lock before a new config is assigned.
-	serverConfigMu.Lock()
-	serverConfig = srvCfg
+	globalServerConfigMu.Lock()
+	globalServerConfig = srvCfg
 	if !globalIsEnvCreds {
-		globalActiveCred = serverConfig.GetCredential()
+		globalActiveCred = globalServerConfig.GetCredential()
 	}
 	if !globalIsEnvBrowser {
-		globalIsBrowserEnabled = serverConfig.GetBrowser()
+		globalIsBrowserEnabled = globalServerConfig.GetBrowser()
 	}
 	if !globalIsEnvRegion {
-		globalServerRegion = serverConfig.GetRegion()
+		globalServerRegion = globalServerConfig.GetRegion()
 	}
-	serverConfigMu.Unlock()
+	if !globalIsEnvDomainName {
+		globalDomainName = globalServerConfig.Domain
+	}
+	if !globalIsStorageClass {
+		globalStandardStorageClass, globalRRStorageClass = globalServerConfig.GetStorageClass()
+	}
+	globalServerConfigMu.Unlock()
 
 	return nil
 }

cmd/config-current_test.go

@@ -34,95 +34,72 @@ func TestServerConfig(t *testing.T) {
 	// remove the root directory after the test ends.
 	defer os.RemoveAll(rootPath)
 
-	if serverConfig.GetRegion() != globalMinioDefaultRegion {
-		t.Errorf("Expecting region `us-east-1` found %s", serverConfig.GetRegion())
+	if globalServerConfig.GetRegion() != globalMinioDefaultRegion {
+		t.Errorf("Expecting region `us-east-1` found %s", globalServerConfig.GetRegion())
 	}
 
 	// Set new region and verify.
-	serverConfig.SetRegion("us-west-1")
-	if serverConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expecting region `us-west-1` found %s", serverConfig.GetRegion())
+	globalServerConfig.SetRegion("us-west-1")
+	if globalServerConfig.GetRegion() != "us-west-1" {
+		t.Errorf("Expecting region `us-west-1` found %s", globalServerConfig.GetRegion())
 	}
 
 	// Set new amqp notification id.
-	serverConfig.Notify.SetAMQPByID("2", amqpNotify{})
-	savedNotifyCfg1 := serverConfig.Notify.GetAMQPByID("2")
+	globalServerConfig.Notify.SetAMQPByID("2", amqpNotify{})
+	savedNotifyCfg1 := globalServerConfig.Notify.GetAMQPByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg1, amqpNotify{}) {
 		t.Errorf("Expecting AMQP config %#v found %#v", amqpNotify{}, savedNotifyCfg1)
 	}
 
 	// Set new elastic search notification id.
-	serverConfig.Notify.SetElasticSearchByID("2", elasticSearchNotify{})
-	savedNotifyCfg2 := serverConfig.Notify.GetElasticSearchByID("2")
+	globalServerConfig.Notify.SetElasticSearchByID("2", elasticSearchNotify{})
+	savedNotifyCfg2 := globalServerConfig.Notify.GetElasticSearchByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg2, elasticSearchNotify{}) {
 		t.Errorf("Expecting Elasticsearch config %#v found %#v", elasticSearchNotify{}, savedNotifyCfg2)
 	}
 
 	// Set new redis notification id.
-	serverConfig.Notify.SetRedisByID("2", redisNotify{})
-	savedNotifyCfg3 := serverConfig.Notify.GetRedisByID("2")
+	globalServerConfig.Notify.SetRedisByID("2", redisNotify{})
+	savedNotifyCfg3 := globalServerConfig.Notify.GetRedisByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg3, redisNotify{}) {
 		t.Errorf("Expecting Redis config %#v found %#v", redisNotify{}, savedNotifyCfg3)
 	}
 
 	// Set new kafka notification id.
-	serverConfig.Notify.SetKafkaByID("2", kafkaNotify{})
-	savedNotifyCfg4 := serverConfig.Notify.GetKafkaByID("2")
+	globalServerConfig.Notify.SetKafkaByID("2", kafkaNotify{})
+	savedNotifyCfg4 := globalServerConfig.Notify.GetKafkaByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg4, kafkaNotify{}) {
 		t.Errorf("Expecting Kafka config %#v found %#v", kafkaNotify{}, savedNotifyCfg4)
 	}
 
 	// Set new Webhook notification id.
-	serverConfig.Notify.SetWebhookByID("2", webhookNotify{})
-	savedNotifyCfg5 := serverConfig.Notify.GetWebhookByID("2")
+	globalServerConfig.Notify.SetWebhookByID("2", webhookNotify{})
+	savedNotifyCfg5 := globalServerConfig.Notify.GetWebhookByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg5, webhookNotify{}) {
 		t.Errorf("Expecting Webhook config %#v found %#v", webhookNotify{}, savedNotifyCfg5)
 	}
 
-	// Set new console logger.
 	// Set new MySQL notification id.
-	serverConfig.Notify.SetMySQLByID("2", mySQLNotify{})
-	savedNotifyCfg6 := serverConfig.Notify.GetMySQLByID("2")
+	globalServerConfig.Notify.SetMySQLByID("2", mySQLNotify{})
+	savedNotifyCfg6 := globalServerConfig.Notify.GetMySQLByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg6, mySQLNotify{}) {
 		t.Errorf("Expecting Webhook config %#v found %#v", mySQLNotify{}, savedNotifyCfg6)
 	}
 
-	// Set new console logger.
 	// Set new MQTT notification id.
-	serverConfig.Notify.SetMQTTByID("2", mqttNotify{})
-	savedNotifyCfg7 := serverConfig.Notify.GetMQTTByID("2")
+	globalServerConfig.Notify.SetMQTTByID("2", mqttNotify{})
+	savedNotifyCfg7 := globalServerConfig.Notify.GetMQTTByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg7, mqttNotify{}) {
 		t.Errorf("Expecting Webhook config %#v found %#v", mqttNotify{}, savedNotifyCfg7)
 	}
 
-	consoleLogger := NewConsoleLogger()
-	serverConfig.Logger.SetConsole(consoleLogger)
-	consoleCfg := serverConfig.Logger.GetConsole()
-	if !reflect.DeepEqual(consoleCfg, consoleLogger) {
-		t.Errorf("Expecting console logger config %#v found %#v", consoleLogger, consoleCfg)
-	}
-	// Set new console logger.
-	consoleLogger.Enable = false
-	serverConfig.Logger.SetConsole(consoleLogger)
-
-	// Set new file logger.
-	fileLogger := NewFileLogger("test-log-file")
-	serverConfig.Logger.SetFile(fileLogger)
-	fileCfg := serverConfig.Logger.GetFile()
-	if !reflect.DeepEqual(fileCfg, fileLogger) {
-		t.Errorf("Expecting file logger config %#v found %#v", fileLogger, fileCfg)
-	}
-	// Set new file logger.
-	fileLogger.Enable = false
-	serverConfig.Logger.SetFile(fileLogger)
-
 	// Match version.
-	if serverConfig.GetVersion() != v19 {
-		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v19)
+	if globalServerConfig.GetVersion() != serverConfigVersion {
+		t.Errorf("Expecting version %s found %s", globalServerConfig.GetVersion(), serverConfigVersion)
 	}
 
 	// Attempt to save.
-	if err := serverConfig.Save(); err != nil {
+	if err := globalServerConfig.Save(); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 
@@ -149,6 +126,9 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	os.Setenv("MINIO_REGION", "us-west-1")
 	defer os.Unsetenv("MINIO_REGION")
 
+	os.Setenv("MINIO_DOMAIN", "domain.com")
+	defer os.Unsetenv("MINIO_DOMAIN")
+
 	defer resetGlobalIsEnvs()
 
 	// Get test root.
@@ -169,17 +149,17 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	defer os.RemoveAll(rootPath)
 
 	// Check if serverConfig has
-	if serverConfig.GetBrowser() {
-		t.Errorf("Expecting browser is set to false found %v", serverConfig.GetBrowser())
+	if globalServerConfig.GetBrowser() {
+		t.Errorf("Expecting browser is set to false found %v", globalServerConfig.GetBrowser())
 	}
 
 	// Check if serverConfig has
-	if serverConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expecting region to be \"us-west-1\" found %v", serverConfig.GetRegion())
+	if globalServerConfig.GetRegion() != "us-west-1" {
+		t.Errorf("Expecting region to be \"us-west-1\" found %v", globalServerConfig.GetRegion())
 	}
 
 	// Check if serverConfig has
-	cred := serverConfig.GetCredential()
+	cred := globalServerConfig.GetCredential()
 
 	if cred.AccessKey != "minio" {
 		t.Errorf("Expecting access key to be `minio` found %s", cred.AccessKey)
@@ -189,6 +169,9 @@ func TestServerConfigWithEnvs(t *testing.T) {
 		t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
 	}
 
+	if globalServerConfig.Domain != "domain.com" {
+		t.Errorf("Expecting Domain to be `domain.com` found " + globalServerConfig.Domain)
+	}
 }
 
 func TestCheckDupJSONKeys(t *testing.T) {
@@ -231,7 +214,7 @@ func TestValidateConfig(t *testing.T) {
 
 	configPath := filepath.Join(rootPath, minioConfigFile)
 
-	v := v19
+	v := serverConfigVersion
 
 	testCases := []struct {
 		configData string
@@ -267,58 +250,55 @@ func TestValidateConfig(t *testing.T) {
 		// Test 10 - duplicated json keys
 		{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
 
-		// Test 11 - empty filename field in File
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "filename": "" } }}`, false},
-
-		// Test 12 - Test AMQP
+		// Test 11 - Test AMQP
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
 
-		// Test 13 - Test NATS
+		// Test 12 - Test NATS
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
 
-		// Test 14 - Test ElasticSearch
+		// Test 13 - Test ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
 
-		// Test 15 - Test Redis
+		// Test 14 - Test Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
 
-		// Test 16 - Test PostgreSQL
+		// Test 15 - Test PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
-		// Test 17 - Test Kafka
+		// Test 16 - Test Kafka
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
 
-		// Test 18 - Test Webhook
+		// Test 17 - Test Webhook
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
 
-		// Test 20 - Test MySQL
+		// Test 18 - Test MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
-		// Test 21 - Test Format for MySQL
+		// Test 19 - Test Format for MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
-		// Test 22 - Test valid Format for MySQL
+		// Test 20 - Test valid Format for MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
-		// Test 23 - Test Format for PostgreSQL
+		// Test 21 - Test Format for PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
-		// Test 24 - Test valid Format for PostgreSQL
+		// Test 22 - Test valid Format for PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
-		// Test 25 - Test Format for ElasticSearch
+		// Test 23 - Test Format for ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
 
-		// Test 26 - Test valid Format for ElasticSearch
+		// Test 24 - Test valid Format for ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
 
-		// Test 27 - Test Format for Redis
+		// Test 25 - Test Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
 
-		// Test 28 - Test valid Format for Redis
+		// Test 26 - Test valid Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
 
-		// Test 29 - Test MQTT
+		// Test 27 - Test MQTT
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
 	}
 
@@ -331,7 +311,7 @@ func TestValidateConfig(t *testing.T) {
 			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, verr)
 		}
 		if !testCase.shouldPass && verr == nil {
-			t.Errorf("Test %d, should fail but it succeed.", i+1)
+			t.Errorf("Test %d, should fail but it succeeded.", i+1)
 		}
 	}
 

cmd/config-migrate.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -142,12 +143,26 @@ func migrateConfig() error {
 		}
 		fallthrough
 	case "18":
-		// Migrate version '17' to '18'.
+		// Migrate version '18' to '19'.
 		if err = migrateV18ToV19(); err != nil {
 			return err
 		}
 		fallthrough
-	case v19:
+	case "19":
+		if err = migrateV19ToV20(); err != nil {
+			return err
+		}
+		fallthrough
+	case "20":
+		if err = migrateV20ToV21(); err != nil {
+			return err
+		}
+		fallthrough
+	case "21":
+		if err = migrateV21ToV22(); err != nil {
+			return err
+		}
+	case serverConfigVersion:
 		// No migration needed. this always points to current version.
 		err = nil
 	}
@@ -190,7 +205,7 @@ func migrateV2ToV3() error {
 		return nil
 	}
 
-	cred, err := createCredential(cv2.Credentials.AccessKey, cv2.Credentials.SecretKey)
+	cred, err := auth.CreateCredentials(cv2.Credentials.AccessKey, cv2.Credentials.SecretKey)
 	if err != nil {
 		return fmt.Errorf("Invalid credential in V2 configuration file. %v", err)
 	}
@@ -1478,3 +1493,325 @@ func migrateV18ToV19() error {
 	log.Printf(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version)
 	return nil
 }
+
+func migrateV19ToV20() error {
+	configFile := getConfigFile()
+
+	cv19 := &serverConfigV19{}
+	_, err := quick.Load(configFile, cv19)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
+	}
+	if cv19.Version != "19" {
+		return nil
+	}
+
+	// Copy over fields from V19 into V20 config struct
+	srvConfig := &serverConfigV20{
+		Logger: &loggers{},
+		Notify: &notifier{},
+	}
+	srvConfig.Version = "20"
+	srvConfig.Credential = cv19.Credential
+	srvConfig.Region = cv19.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	srvConfig.Logger.Console = cv19.Logger.Console
+	srvConfig.Logger.File = cv19.Logger.File
+
+	// check and set notifiers config
+	if len(cv19.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		// New deliveryMode parameter is added for AMQP,
+		// default value is already 0, so nothing to
+		// explicitly migrate here.
+		srvConfig.Notify.AMQP = cv19.Notify.AMQP
+	}
+	if len(cv19.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv19.Notify.ElasticSearch
+	}
+	if len(cv19.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]redisNotify)
+		srvConfig.Notify.Redis["1"] = redisNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv19.Notify.Redis
+	}
+	if len(cv19.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
+		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv19.Notify.PostgreSQL
+	}
+	if len(cv19.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
+		srvConfig.Notify.Kafka["1"] = kafkaNotify{}
+	} else {
+		srvConfig.Notify.Kafka = cv19.Notify.Kafka
+	}
+	if len(cv19.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]natsNotify)
+		srvConfig.Notify.NATS["1"] = natsNotify{}
+	} else {
+		srvConfig.Notify.NATS = cv19.Notify.NATS
+	}
+	if len(cv19.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]webhookNotify)
+		srvConfig.Notify.Webhook["1"] = webhookNotify{}
+	} else {
+		srvConfig.Notify.Webhook = cv19.Notify.Webhook
+	}
+	if len(cv19.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
+		srvConfig.Notify.MySQL["1"] = mySQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv19.Notify.MySQL
+	}
+	if len(cv19.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]mqttNotify)
+		srvConfig.Notify.MQTT["1"] = mqttNotify{}
+	} else {
+		srvConfig.Notify.MQTT = cv19.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv19.Browser
+
+	if err = quick.Save(configFile, srvConfig); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err)
+	}
+
+	log.Printf(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version)
+	return nil
+}
+
+func migrateV20ToV21() error {
+	configFile := getConfigFile()
+
+	cv20 := &serverConfigV20{}
+	_, err := quick.Load(configFile, cv20)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘20’. %v", err)
+	}
+	if cv20.Version != "20" {
+		return nil
+	}
+
+	// Copy over fields from V20 into V21 config struct
+	srvConfig := &serverConfigV21{
+		Notify: &notifier{},
+	}
+	srvConfig.Version = "21"
+	srvConfig.Credential = cv20.Credential
+	srvConfig.Region = cv20.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	// check and set notifiers config
+	if len(cv20.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		// New deliveryMode parameter is added for AMQP,
+		// default value is already 0, so nothing to
+		// explicitly migrate here.
+		srvConfig.Notify.AMQP = cv20.Notify.AMQP
+	}
+	if len(cv20.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv20.Notify.ElasticSearch
+	}
+	if len(cv20.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]redisNotify)
+		srvConfig.Notify.Redis["1"] = redisNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv20.Notify.Redis
+	}
+	if len(cv20.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
+		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv20.Notify.PostgreSQL
+	}
+	if len(cv20.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
+		srvConfig.Notify.Kafka["1"] = kafkaNotify{}
+	} else {
+		srvConfig.Notify.Kafka = cv20.Notify.Kafka
+	}
+	if len(cv20.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]natsNotify)
+		srvConfig.Notify.NATS["1"] = natsNotify{}
+	} else {
+		srvConfig.Notify.NATS = cv20.Notify.NATS
+	}
+	if len(cv20.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]webhookNotify)
+		srvConfig.Notify.Webhook["1"] = webhookNotify{}
+	} else {
+		srvConfig.Notify.Webhook = cv20.Notify.Webhook
+	}
+	if len(cv20.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
+		srvConfig.Notify.MySQL["1"] = mySQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv20.Notify.MySQL
+	}
+	if len(cv20.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]mqttNotify)
+		srvConfig.Notify.MQTT["1"] = mqttNotify{}
+	} else {
+		srvConfig.Notify.MQTT = cv20.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv20.Browser
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv20.Domain
+
+	if err = quick.Save(configFile, srvConfig); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv20.Version, srvConfig.Version, err)
+	}
+
+	log.Printf(configMigrateMSGTemplate, configFile, cv20.Version, srvConfig.Version)
+	return nil
+}
+
+func migrateV21ToV22() error {
+	configFile := getConfigFile()
+
+	cv21 := &serverConfigV21{}
+	_, err := quick.Load(configFile, cv21)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘21’. %v", err)
+	}
+	if cv21.Version != "21" {
+		return nil
+	}
+
+	// Copy over fields from V21 into V22 config struct
+	srvConfig := &serverConfigV22{
+		Notify: &notifier{},
+	}
+	srvConfig.Version = serverConfigVersion
+	srvConfig.Credential = cv21.Credential
+	srvConfig.Region = cv21.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	// check and set notifiers config
+	if len(cv21.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		// New deliveryMode parameter is added for AMQP,
+		// default value is already 0, so nothing to
+		// explicitly migrate here.
+		srvConfig.Notify.AMQP = cv21.Notify.AMQP
+	}
+	if len(cv21.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv21.Notify.ElasticSearch
+	}
+	if len(cv21.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]redisNotify)
+		srvConfig.Notify.Redis["1"] = redisNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv21.Notify.Redis
+	}
+	if len(cv21.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
+		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv21.Notify.PostgreSQL
+	}
+	if len(cv21.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
+		srvConfig.Notify.Kafka["1"] = kafkaNotify{}
+	} else {
+		srvConfig.Notify.Kafka = cv21.Notify.Kafka
+	}
+	if len(cv21.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]natsNotify)
+		srvConfig.Notify.NATS["1"] = natsNotify{}
+	} else {
+		srvConfig.Notify.NATS = cv21.Notify.NATS
+	}
+	if len(cv21.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]webhookNotify)
+		srvConfig.Notify.Webhook["1"] = webhookNotify{}
+	} else {
+		srvConfig.Notify.Webhook = cv21.Notify.Webhook
+	}
+	if len(cv21.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
+		srvConfig.Notify.MySQL["1"] = mySQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv21.Notify.MySQL
+	}
+	if len(cv21.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]mqttNotify)
+		srvConfig.Notify.MQTT["1"] = mqttNotify{}
+	} else {
+		srvConfig.Notify.MQTT = cv21.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv21.Browser
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv21.Domain
+
+	if err = quick.Save(configFile, srvConfig); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv21.Version, srvConfig.Version, err)
+	}
+
+	log.Printf(configMigrateMSGTemplate, configFile, cv21.Version, srvConfig.Version)
+	return nil
+}

cmd/config-migrate_test.go

@@ -125,11 +125,16 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	if err := migrateV18ToV19(); err != nil {
 		t.Fatal("migrate v18 to v19 should succeed when no config file is found")
 	}
-
+	if err := migrateV19ToV20(); err != nil {
+		t.Fatal("migrate v19 to v20 should succeed when no config file is found")
+	}
+	if err := migrateV20ToV21(); err != nil {
+		t.Fatal("migrate v20 to v21 should succeed when no config file is found")
+	}
 }
 
-// Test if a config migration from v2 to v19 is successfully done
-func TestServerConfigMigrateV2toV19(t *testing.T) {
+// Test if a config migration from v2 to v21 is successfully done
+func TestServerConfigMigrateV2toV21(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Init Test config failed")
@@ -169,17 +174,17 @@ func TestServerConfigMigrateV2toV19(t *testing.T) {
 	}
 
 	// Check the version number in the upgraded config file
-	expectedVersion := v19
-	if serverConfig.Version != expectedVersion {
-		t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
+	expectedVersion := serverConfigVersion
+	if globalServerConfig.Version != expectedVersion {
+		t.Fatalf("Expect version "+expectedVersion+", found: %v", globalServerConfig.Version)
 	}
 
 	// Check if accessKey and secretKey are not altered during migration
-	if serverConfig.Credential.AccessKey != accessKey {
-		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, serverConfig.Credential.AccessKey)
+	if globalServerConfig.Credential.AccessKey != accessKey {
+		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, globalServerConfig.Credential.AccessKey)
 	}
-	if serverConfig.Credential.SecretKey != secretKey {
-		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, serverConfig.Credential.SecretKey)
+	if globalServerConfig.Credential.SecretKey != secretKey {
+		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, globalServerConfig.Credential.SecretKey)
 	}
 }
 
@@ -252,6 +257,12 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV18ToV19(); err == nil {
 		t.Fatal("migrateConfigV18ToV19() should fail with a corrupted json")
 	}
+	if err := migrateV19ToV20(); err == nil {
+		t.Fatal("migrateConfigV19ToV20() should fail with a corrupted json")
+	}
+	if err := migrateV20ToV21(); err == nil {
+		t.Fatal("migrateConfigV20ToV21() should fail with a corrupted json")
+	}
 }
 
 // Test if all migrate code returns error with corrupted config files

cmd/config-versions.go

@@ -16,7 +16,11 @@
 
 package cmd
 
-import "sync"
+import (
+	"sync"
+
+	"github.com/minio/minio/pkg/auth"
+)
 
 /////////////////// Config V1 ///////////////////
 type configV1 struct {
@@ -92,8 +96,8 @@ type configV3 struct {
 	Addr string `json:"address"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV3 `json:"logger"`
@@ -122,8 +126,8 @@ type configV4 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV4 `json:"logger"`
@@ -179,8 +183,8 @@ type configV5 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV5 `json:"logger"`
@@ -209,8 +213,8 @@ type configV6 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -246,8 +250,8 @@ type serverConfigV7 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -265,8 +269,8 @@ type serverConfigV8 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -284,8 +288,8 @@ type serverConfigV9 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -310,8 +314,8 @@ type serverConfigV10 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV7 `json:"logger"`
@@ -338,8 +342,8 @@ type serverConfigV11 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV7 `json:"logger"`
@@ -354,8 +358,8 @@ type serverConfigV12 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV7 `json:"logger"`
@@ -370,8 +374,8 @@ type serverConfigV13 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -386,9 +390,9 @@ type serverConfigV14 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential  `json:"credential"`
-	Region     string      `json:"region"`
-	Browser    BrowserFlag `json:"browser"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -403,9 +407,9 @@ type serverConfigV15 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential  `json:"credential"`
-	Region     string      `json:"region"`
-	Browser    BrowserFlag `json:"browser"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -414,15 +418,21 @@ type serverConfigV15 struct {
 	Notify *notifier `json:"notify"`
 }
 
+type loggers struct {
+	sync.RWMutex
+	Console ConsoleLogger `json:"console"`
+	File    FileLogger    `json:"file"`
+}
+
 // serverConfigV16 server configuration version '16' which is like
 // version '15' except it makes a change to logging configuration.
 type serverConfigV16 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential  `json:"credential"`
-	Region     string      `json:"region"`
-	Browser    BrowserFlag `json:"browser"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -439,9 +449,9 @@ type serverConfigV17 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential  `json:"credential"`
-	Region     string      `json:"region"`
-	Browser    BrowserFlag `json:"browser"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -458,9 +468,9 @@ type serverConfigV18 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential  `json:"credential"`
-	Region     string      `json:"region"`
-	Browser    BrowserFlag `json:"browser"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -468,3 +478,74 @@ type serverConfigV18 struct {
 	// Notification queue configuration.
 	Notify *notifier `json:"notify"`
 }
+
+// serverConfigV19 server configuration version '19' which is like
+// version '18' except it adds support for MQTT notifications.
+type serverConfigV19 struct {
+	sync.RWMutex
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV20 server configuration version '20' which is like
+// version '19' except it adds support for VirtualHostDomain
+type serverConfigV20 struct {
+	sync.RWMutex
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+	Domain     string           `json:"domain"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV21 is just like version '20' without logger field
+type serverConfigV21 struct {
+	sync.RWMutex
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+	Domain     string           `json:"domain"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV22 is just like version '21' with added support
+// for StorageClass
+type serverConfigV22 struct {
+	sync.RWMutex
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}

cmd/credential_test.go

@@ -1,100 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import "testing"
-
-func TestMustGetNewCredential(t *testing.T) {
-	cred := mustGetNewCredential()
-	if !cred.IsValid() {
-		t.Fatalf("Failed to get new valid credential")
-	}
-	if len(cred.SecretKey) != secretKeyMaxLen {
-		t.Fatalf("Invalid length %d of the secretKey credential generated, expected %d", len(cred.SecretKey), secretKeyMaxLen)
-	}
-}
-
-func TestCreateCredential(t *testing.T) {
-	cred := mustGetNewCredential()
-	testCases := []struct {
-		accessKey      string
-		secretKey      string
-		expectedResult bool
-		expectedErr    error
-	}{
-		// Access key too small (min 5 chars).
-		{"user", "pass", false, errInvalidAccessKeyLength},
-		// Long access key is ok.
-		{"user123456789012345678901234567890", "password", true, nil},
-		// Secret key too small (min 8 chars).
-		{"myuser", "pass", false, errInvalidSecretKeyLength},
-		// Long secret key is ok.
-		{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", true, nil},
-		{"myuser", "mypassword", true, nil},
-		{cred.AccessKey, cred.SecretKey, true, nil},
-	}
-
-	for _, testCase := range testCases {
-		cred, err := createCredential(testCase.accessKey, testCase.secretKey)
-		if testCase.expectedErr == nil {
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
-			}
-		} else if err == nil {
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
-		}
-
-		if testCase.expectedResult != cred.IsValid() {
-			t.Fatalf("cred: expected: %v, got: %v", testCase.expectedResult, cred.IsValid())
-		}
-	}
-}
-
-func TestCredentialEqual(t *testing.T) {
-	cred := mustGetNewCredential()
-	testCases := []struct {
-		cred           credential
-		ccred          credential
-		expectedResult bool
-	}{
-		// Empty compare credential
-		{cred, credential{}, false},
-		// Empty credential
-		{credential{}, cred, false},
-		// Two different credentials
-		{cred, mustGetNewCredential(), false},
-		// Access key is different in compare credential.
-		{cred, credential{AccessKey: "myuser", SecretKey: cred.SecretKey}, false},
-		// Secret key is different in compare credential.
-		{cred, credential{AccessKey: cred.AccessKey, SecretKey: "mypassword"}, false},
-		// secretHashKey is missing in compare credential.
-		{cred, credential{AccessKey: cred.AccessKey, SecretKey: cred.SecretKey}, true},
-		// secretHashKey is missing in credential.
-		{credential{AccessKey: cred.AccessKey, SecretKey: cred.SecretKey}, cred, true},
-		// Same credentials.
-		{cred, cred, true},
-	}
-
-	for _, testCase := range testCases {
-		result := testCase.cred.Equal(testCase.ccred)
-		if result != testCase.expectedResult {
-			t.Fatalf("cred: expected: %v, got: %v", testCase.expectedResult, result)
-		}
-	}
-}

cmd/encryption-v1.go

@@ -0,0 +1,339 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+	"io"
+	"net/http"
+
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+var (
+	// AWS errors for invalid SSE-C requests.
+	errInsecureSSERequest  = errors.New("Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection")
+	errEncryptedObject     = errors.New("The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object")
+	errInvalidSSEAlgorithm = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm")
+	errMissingSSEKey       = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key")
+	errInvalidSSEKey       = errors.New("The secret key was invalid for the specified algorithm")
+	errMissingSSEKeyMD5    = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key")
+	errSSEKeyMD5Mismatch   = errors.New("The calculated MD5 hash of the key did not match the hash that was provided")
+	errSSEKeyMismatch      = errors.New("The client provided key does not match the key provided when the object was encrypted") // this msg is not shown to the client
+
+	// Additional Minio errors for SSE-C requests.
+	errObjectTampered = errors.New("The requested object was modified and may be compromised")
+)
+
+const (
+	// SSECustomerAlgorithm is the AWS SSE-C algorithm HTTP header key.
+	SSECustomerAlgorithm = "X-Amz-Server-Side-Encryption-Customer-Algorithm"
+	// SSECustomerKey is the AWS SSE-C encryption key HTTP header key.
+	SSECustomerKey = "X-Amz-Server-Side-Encryption-Customer-Key"
+	// SSECustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key.
+	SSECustomerKeyMD5 = "X-Amz-Server-Side-Encryption-Customer-Key-MD5"
+)
+
+const (
+	// SSECustomerKeySize is the size of valid client provided encryption keys in bytes.
+	// Currently AWS supports only AES256. So the SSE-C key size is fixed to 32 bytes.
+	SSECustomerKeySize = 32
+
+	// SSECustomerAlgorithmAES256 the only valid S3 SSE-C encryption algorithm identifier.
+	SSECustomerAlgorithmAES256 = "AES256"
+)
+
+// SSE-C key derivation, key verification and key update:
+// H: Hash function [32 = |H(m)|]
+// AE: authenticated encryption scheme, AD: authenticated decryption scheme [m = AD(k, AE(k, m))]
+//
+// Key derivation:
+// 	 Input:
+// 		key     := 32 bytes              # client provided key
+// 		Re, Rm  := 32 bytes, 32 bytes    # uniformly random
+//
+//   Seal:
+// 		k       := H(key || Re)          # object encryption key
+// 		r       := H(Rm)                 # save as object metadata [ServerSideEncryptionIV]
+//		KeK     := H(key || r)           # key encryption key
+// 		K       := AE(KeK, k)            # save as object metadata [ServerSideEncryptionSealedKey]
+// ------------------------------------------------------------------------------------------------
+// Key verification:
+// 	 Input:
+// 		key     := 32 bytes              # client provided key
+// 		r       := 32 bytes              # object metadata [ServerSideEncryptionIV]
+// 		K       := 32 bytes              # object metadata [ServerSideEncryptionSealedKey]
+//
+//   Open:
+//		KeK     := H(key || r)           # key encryption key
+//		k       := AD(Kek, K)            # object encryption key
+// -------------------------------------------------------------------------------------------------
+// Key update:
+// 	 Input:
+// 		key     := 32 bytes              # old client provided key
+// 		key'    := 32 bytes              # new client provided key
+// 		Rm      := 32 bytes              # uniformly random
+// 		r       := 32 bytes              # object metadata [ServerSideEncryptionIV]
+// 		K       := 32 bytes              # object metadata [ServerSideEncryptionSealedKey]
+//
+//   Update:
+//      1. open:
+//			KeK     := H(key || r)       # key encryption key
+//			k       := AD(Kek, K)        # object encryption key
+//      2. seal:
+//			r'      := H(Rm)			 # save as object metadata [ServerSideEncryptionIV]
+//			KeK'    := H(key' || r')	 # new key encryption key
+// 			K'      := AE(KeK', k)       # save as object metadata [ServerSideEncryptionSealedKey]
+
+const (
+	// ServerSideEncryptionIV is a 32 byte randomly generated IV used to derive an
+	// unique key encryption key from the client provided key. The combination of this value
+	// and the client-provided key MUST be unique.
+	ServerSideEncryptionIV = ReservedMetadataPrefix + "Server-Side-Encryption-Iv"
+
+	// ServerSideEncryptionSealAlgorithm identifies a combination of a cryptographic hash function and
+	// an authenticated en/decryption scheme to seal the object encryption key.
+	ServerSideEncryptionSealAlgorithm = ReservedMetadataPrefix + "Server-Side-Encryption-Seal-Algorithm"
+
+	// ServerSideEncryptionSealedKey is the sealed object encryption key. The sealed key can be decrypted
+	// by the key encryption key derived from the client provided key and the server-side-encryption IV.
+	ServerSideEncryptionSealedKey = ReservedMetadataPrefix + "Server-Side-Encryption-Sealed-Key"
+)
+
+// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
+// hash function.
+const SSESealAlgorithmDareSha256 = "DARE-SHA256"
+
+// IsSSECustomerRequest returns true if the given HTTP header
+// contains server-side-encryption with customer provided key fields.
+func IsSSECustomerRequest(header http.Header) bool {
+	return header.Get(SSECustomerAlgorithm) != "" || header.Get(SSECustomerKey) != "" || header.Get(SSECustomerKeyMD5) != ""
+}
+
+// ParseSSECustomerRequest parses the SSE-C header fields of the provided request.
+// It returns the client provided key on success.
+func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) {
+	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
+		// we cannot use r.TLS == nil here because Go's http implementation reflects on
+		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
+		// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
+		// will always fail -> r.TLS is always nil even for TLS requests.
+		return nil, errInsecureSSERequest
+	}
+	header := r.Header
+	if algorithm := header.Get(SSECustomerAlgorithm); algorithm != SSECustomerAlgorithmAES256 {
+		return nil, errInvalidSSEAlgorithm
+	}
+	if header.Get(SSECustomerKey) == "" {
+		return nil, errMissingSSEKey
+	}
+	if header.Get(SSECustomerKeyMD5) == "" {
+		return nil, errMissingSSEKeyMD5
+	}
+
+	key, err = base64.StdEncoding.DecodeString(header.Get(SSECustomerKey))
+	if err != nil {
+		return nil, errInvalidSSEKey
+	}
+	header.Del(SSECustomerKey) // make sure we do not save the key by accident
+
+	if len(key) != SSECustomerKeySize {
+		return nil, errInvalidSSEKey
+	}
+
+	keyMD5, err := base64.StdEncoding.DecodeString(header.Get(SSECustomerKeyMD5))
+	if err != nil {
+		return nil, errSSEKeyMD5Mismatch
+	}
+	if md5Sum := md5.Sum(key); !bytes.Equal(md5Sum[:], keyMD5) {
+		return nil, errSSEKeyMD5Mismatch
+	}
+	return key, nil
+}
+
+// EncryptRequest takes the client provided content and encrypts the data
+// with the client provided key. It also marks the object as client-side-encrypted
+// and sets the correct headers.
+func EncryptRequest(content io.Reader, r *http.Request, metadata map[string]string) (io.Reader, error) {
+	key, err := ParseSSECustomerRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
+
+	// security notice:
+	//  - If the first 32 bytes of the random value are ever repeated under the same client-provided
+	//    key the encrypted object will not be tamper-proof. [ P(coll) ~= 1 / 2^(256 / 2)]
+	//  - If the last 32 bytes of the random value are ever repeated under the same client-provided
+	//    key an adversary may be able to extract the object encryption key. This depends on the
+	//    authenticated en/decryption scheme. The DARE format will generate an 8 byte nonce which must
+	//    be repeated in addition to reveal the object encryption key.
+	//    [ P(coll) ~= 1 / 2^((256 + 64) / 2) ]
+	nonce := make([]byte, 64) // generate random values for key derivation
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		return nil, err
+	}
+	sha := sha256.New() // derive object encryption key
+	sha.Write(key)
+	sha.Write(nonce[:32])
+	objectEncryptionKey := sha.Sum(nil)
+
+	iv := sha256.Sum256(nonce[32:]) // derive key encryption key
+	sha = sha256.New()
+	sha.Write(key)
+	sha.Write(iv[:])
+	keyEncryptionKey := sha.Sum(nil)
+
+	sealedKey := bytes.NewBuffer(nil) // sealedKey := 16 byte header + 32 byte payload + 16 byte tag
+	n, err := sio.Encrypt(sealedKey, bytes.NewReader(objectEncryptionKey), sio.Config{
+		Key: keyEncryptionKey,
+	})
+	if n != 64 || err != nil {
+		return nil, errors.New("failed to seal object encryption key") // if this happens there's a bug in the code (may panic ?)
+	}
+
+	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey})
+	if err != nil {
+		return nil, errInvalidSSEKey
+	}
+
+	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(iv[:])
+	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareSha256
+	metadata[ServerSideEncryptionSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Bytes())
+	return reader, nil
+}
+
+// DecryptRequest decrypts the object with the client provided key. It also removes
+// the client-side-encryption metadata from the object and sets the correct headers.
+func DecryptRequest(client io.Writer, r *http.Request, metadata map[string]string) (io.WriteCloser, error) {
+	key, err := ParseSSECustomerRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
+
+	if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
+		return nil, errObjectTampered
+	}
+	iv, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionIV])
+	if err != nil || len(iv) != 32 {
+		return nil, errObjectTampered
+	}
+	sealedKey, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionSealedKey])
+	if err != nil || len(sealedKey) != 64 {
+		return nil, errObjectTampered
+	}
+
+	sha := sha256.New() // derive key encryption key
+	sha.Write(key)
+	sha.Write(iv)
+	keyEncryptionKey := sha.Sum(nil)
+
+	objectEncryptionKey := bytes.NewBuffer(nil) // decrypt object encryption key
+	n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
+		Key: keyEncryptionKey,
+	})
+	if n != 32 || err != nil {
+		// Either the provided key does not match or the object was tampered.
+		// To provide strict AWS S3 compatibility we return: access denied.
+		return nil, errSSEKeyMismatch
+	}
+
+	writer, err := sio.DecryptWriter(client, sio.Config{Key: objectEncryptionKey.Bytes()})
+	if err != nil {
+		return nil, errInvalidSSEKey
+	}
+
+	delete(metadata, ServerSideEncryptionIV)
+	delete(metadata, ServerSideEncryptionSealAlgorithm)
+	delete(metadata, ServerSideEncryptionSealedKey)
+	return writer, nil
+}
+
+// IsEncrypted returns true if the object is marked as encrypted.
+func (o *ObjectInfo) IsEncrypted() bool {
+	if _, ok := o.UserDefined[ServerSideEncryptionIV]; ok {
+		return true
+	}
+	if _, ok := o.UserDefined[ServerSideEncryptionSealAlgorithm]; ok {
+		return true
+	}
+	if _, ok := o.UserDefined[ServerSideEncryptionSealedKey]; ok {
+		return true
+	}
+	return false
+}
+
+// DecryptedSize returns the size of the object after decryption in bytes.
+// It returns an error if the object is not encrypted or marked as encrypted
+// but has an invalid size.
+// DecryptedSize panics if the referred object is not encrypted.
+func (o *ObjectInfo) DecryptedSize() (int64, error) {
+	if !o.IsEncrypted() {
+		panic("cannot compute decrypted size of an object which is not encrypted")
+	}
+	if o.Size == 0 {
+		return o.Size, nil
+	}
+	size := (o.Size / (32 + 64*1024)) * (64 * 1024)
+	if mod := o.Size % (32 + 64*1024); mod > 0 {
+		if mod < 33 {
+			return -1, errObjectTampered // object is not 0 size but smaller than the smallest valid encrypted object
+		}
+		size += mod - 32
+	}
+	return size, nil
+}
+
+// EncryptedSize returns the size of the object after encryption.
+// An encrypted object is always larger than a plain object
+// except for zero size objects.
+func (o *ObjectInfo) EncryptedSize() int64 {
+	size := (o.Size / (64 * 1024)) * (32 + 64*1024)
+	if mod := o.Size % (64 * 1024); mod > 0 {
+		size += mod + 32
+	}
+	return size
+}
+
+// DecryptObjectInfo tries to decrypt the provided object if it is encrypted.
+// It fails if the object is encrypted and the HTTP headers don't contain
+// SSE-C headers or the object is not encrypted but SSE-C headers are provided. (AWS behavior)
+// DecryptObjectInfo returns 'ErrNone' if the object is not encrypted or the
+// decryption succeeded.
+//
+// DecryptObjectInfo also returns whether the object is encrypted or not.
+func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErrorCode, encrypted bool) {
+	if apiErr, encrypted = ErrNone, info.IsEncrypted(); !encrypted && IsSSECustomerRequest(headers) {
+		apiErr = ErrInvalidEncryptionParameters
+	} else if encrypted {
+		if !IsSSECustomerRequest(headers) {
+			apiErr = ErrSSEEncryptedObject
+			return
+		}
+		var err error
+		if info.Size, err = info.DecryptedSize(); err != nil {
+			apiErr = toAPIErrorCode(err)
+		}
+	}
+	return
+}

cmd/encryption-v1_test.go

@@ -0,0 +1,390 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"net/http"
+	"testing"
+)
+
+var isSSECustomerRequestTests = []struct {
+	headers    map[string]string
+	sseRequest bool
+}{
+	{headers: map[string]string{SSECustomerAlgorithm: "AES256", SSECustomerKey: "key", SSECustomerKeyMD5: "md5"}, sseRequest: true},                    // 0
+	{headers: map[string]string{SSECustomerAlgorithm: "AES256"}, sseRequest: true},                                                                     // 1
+	{headers: map[string]string{SSECustomerKey: "key"}, sseRequest: true},                                                                              // 2
+	{headers: map[string]string{SSECustomerKeyMD5: "md5"}, sseRequest: true},                                                                           // 3
+	{headers: map[string]string{}, sseRequest: false},                                                                                                  // 4
+	{headers: map[string]string{SSECustomerAlgorithm + " ": "AES256", " " + SSECustomerKey: "key", SSECustomerKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
+	{headers: map[string]string{SSECustomerAlgorithm: "", SSECustomerKey: "", SSECustomerKeyMD5: ""}, sseRequest: false},                               // 6
+}
+
+func TestIsSSECustomerRequest(t *testing.T) {
+	for i, test := range isSSECustomerRequestTests {
+		headers := http.Header{}
+		for k, v := range test.headers {
+			headers.Set(k, v)
+		}
+		if IsSSECustomerRequest(headers) != test.sseRequest {
+			t.Errorf("Test %d: Expected IsSSECustomerRequest to return %v", i, test.sseRequest)
+		}
+	}
+}
+
+var parseSSECustomerRequestTests = []struct {
+	headers map[string]string
+	useTLS  bool
+	err     error
+}{
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		useTLS: true, err: nil,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		useTLS: false, err: errInsecureSSERequest,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES 256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		useTLS: true, err: errInvalidSSEAlgorithm,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
+			SSECustomerKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
+		},
+		useTLS: true, err: errSSEKeyMD5Mismatch,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
+			SSECustomerKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
+		},
+		useTLS: true, err: errInvalidSSEKey,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
+			SSECustomerKeyMD5:    " +jq/LwEOEO90YtiTuNFVw==",
+		},
+		useTLS: true, err: errSSEKeyMD5Mismatch,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
+			SSECustomerKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
+		},
+		useTLS: true, err: errInvalidSSEKey,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "", // 7
+			SSECustomerKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
+		},
+		useTLS: true, err: errMissingSSEKey,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
+			SSECustomerKeyMD5:    "",
+		},
+		useTLS: true, err: errMissingSSEKeyMD5,
+	},
+}
+
+func TestParseSSECustomerRequest(t *testing.T) {
+	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
+	for i, test := range parseSSECustomerRequestTests {
+		headers := http.Header{}
+		for k, v := range test.headers {
+			headers.Set(k, v)
+		}
+		request := &http.Request{}
+		request.Header = headers
+		globalIsSSL = test.useTLS
+
+		_, err := ParseSSECustomerRequest(request)
+		if err != test.err {
+			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
+		}
+		key := request.Header.Get(SSECustomerKey)
+		if (err == nil || err == errSSEKeyMD5Mismatch) && key != "" {
+			t.Errorf("Test %d: Client key survived parsing - found key: %v", i, key)
+		}
+	}
+}
+
+var encryptedSizeTests = []struct {
+	size, encsize int64
+}{
+	{size: 0, encsize: 0},                                           // 0
+	{size: 1, encsize: 33},                                          // 1
+	{size: 1024, encsize: 1024 + 32},                                // 2
+	{size: 2 * 64 * 1024, encsize: 2 * (64*1024 + 32)},              // 3
+	{size: 100*64*1024 + 1, encsize: 100*(64*1024+32) + 33},         // 4
+	{size: 64*1024 + 1, encsize: (64*1024 + 32) + 33},               // 5
+	{size: 5 * 1024 * 1024 * 1024, encsize: 81920 * (64*1024 + 32)}, // 6
+}
+
+func TestEncryptedSize(t *testing.T) {
+	for i, test := range encryptedSizeTests {
+		objInfo := ObjectInfo{Size: test.size}
+		if size := objInfo.EncryptedSize(); test.encsize != size {
+			t.Errorf("Test %d: got encrypted size: #%d want: #%d", i, size, test.encsize)
+		}
+	}
+}
+
+var decryptSSECustomerObjectInfoTests = []struct {
+	encsize, size int64
+	err           error
+}{
+	{encsize: 0, size: 0, err: nil},                                           // 0
+	{encsize: 33, size: 1, err: nil},                                          // 1
+	{encsize: 1024 + 32, size: 1024, err: nil},                                // 2
+	{encsize: 2 * (64*1024 + 32), size: 2 * 64 * 1024, err: nil},              // 3
+	{encsize: 100*(64*1024+32) + 33, size: 100*64*1024 + 1, err: nil},         // 4
+	{encsize: (64*1024 + 32) + 33, size: 64*1024 + 1, err: nil},               // 5
+	{encsize: 81920 * (64*1024 + 32), size: 5 * 1024 * 1024 * 1024, err: nil}, // 6
+	{encsize: 0, size: 0, err: nil},                                           // 7
+	{encsize: 64*1024 + 32 + 31, size: 0, err: errObjectTampered},             // 8
+}
+
+func TestDecryptedSize(t *testing.T) {
+	for i, test := range decryptSSECustomerObjectInfoTests {
+		objInfo := ObjectInfo{Size: test.encsize}
+		objInfo.UserDefined = map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
+		}
+
+		size, err := objInfo.DecryptedSize()
+		if err != test.err || (size != test.size && err == nil) {
+			t.Errorf("Test %d: decryption returned: %v want: %v", i, err, test.err)
+		}
+		if err == nil && size != test.size {
+			t.Errorf("Test %d: got decrypted size: #%d want: #%d", i, size, test.size)
+		}
+	}
+}
+
+var encryptRequestTests = []struct {
+	header   map[string]string
+	metadata map[string]string
+}{
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		metadata: map[string]string{},
+	},
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		metadata: map[string]string{
+			SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+		},
+	},
+}
+
+func TestEncryptRequest(t *testing.T) {
+	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
+	globalIsSSL = true
+	for i, test := range encryptRequestTests {
+		content := bytes.NewReader(make([]byte, 64))
+		req := &http.Request{Header: http.Header{}}
+		for k, v := range test.header {
+			req.Header.Set(k, v)
+		}
+		_, err := EncryptRequest(content, req, test.metadata)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
+		}
+		if key, ok := test.metadata[SSECustomerKey]; ok {
+			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
+		}
+		if kdf, ok := test.metadata[ServerSideEncryptionSealAlgorithm]; !ok {
+			t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf)
+		}
+		if iv, ok := test.metadata[ServerSideEncryptionIV]; !ok {
+			t.Errorf("Test %d: ServerSideEncryptionIV must be part of metadata: %v", i, iv)
+		}
+		if mac, ok := test.metadata[ServerSideEncryptionSealedKey]; !ok {
+			t.Errorf("Test %d: ServerSideEncryptionKeyMAC must be part of metadata: %v", i, mac)
+		}
+	}
+}
+
+var decryptRequestTests = []struct {
+	header     map[string]string
+	metadata   map[string]string
+	shouldFail bool
+}{
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
+			SSECustomerKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
+			ServerSideEncryptionIV:            "7nQqotA8xgrPx6QK7Ap3GCfjKitqJSrGP7xzgErSJlw=",
+			ServerSideEncryptionSealedKey:     "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA==",
+		},
+		shouldFail: false,
+	},
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: "HMAC-SHA3",
+			ServerSideEncryptionIV:            "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			ServerSideEncryptionSealedKey:     "SY5E9AvI2tI7/nUrUAssIGE32Hcs4rR9z/CUuPqu5N4=",
+		},
+		shouldFail: true,
+	},
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
+			ServerSideEncryptionIV:            "RrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			ServerSideEncryptionSealedKey:     "SY5E9AvI2tI7/nUrUAssIGE32Hcs4rR9z/CUuPqu5N4=",
+		},
+		shouldFail: true,
+	},
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
+			ServerSideEncryptionIV:            "XAm0dRrJsEsyPb1UuFNezv1bl9ehxuYsgUVC/MUctE2k=",
+			ServerSideEncryptionSealedKey:     "SY5E9AvI2tI7/nUrUAssIGE32Hds4rR9z/CUuPqu5N4=",
+		},
+		shouldFail: true,
+	},
+}
+
+func TestDecryptRequest(t *testing.T) {
+	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
+	globalIsSSL = true
+	for i, test := range decryptRequestTests {
+		client := bytes.NewBuffer(nil)
+		req := &http.Request{Header: http.Header{}}
+		for k, v := range test.header {
+			req.Header.Set(k, v)
+		}
+		_, err := DecryptRequest(client, req, test.metadata)
+		if err != nil && !test.shouldFail {
+			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
+		}
+		if key, ok := test.metadata[SSECustomerKey]; ok {
+			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
+		}
+		if kdf, ok := test.metadata[ServerSideEncryptionSealAlgorithm]; ok && !test.shouldFail {
+			t.Errorf("Test %d: ServerSideEncryptionKDF should not be part of metadata: %v", i, kdf)
+		}
+		if iv, ok := test.metadata[ServerSideEncryptionIV]; ok && !test.shouldFail {
+			t.Errorf("Test %d: ServerSideEncryptionIV should not be part of metadata: %v", i, iv)
+		}
+		if mac, ok := test.metadata[ServerSideEncryptionSealedKey]; ok && !test.shouldFail {
+			t.Errorf("Test %d: ServerSideEncryptionKeyMAC should not be part of metadata: %v", i, mac)
+		}
+	}
+}
+
+var decryptObjectInfoTests = []struct {
+	info    ObjectInfo
+	headers http.Header
+	expErr  APIErrorCode
+}{
+	{
+		info:    ObjectInfo{Size: 100},
+		headers: http.Header{},
+		expErr:  ErrNone,
+	},
+	{
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
+		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
+		expErr:  ErrNone,
+	},
+	{
+		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
+		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
+		expErr:  ErrNone,
+	},
+	{
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
+		headers: http.Header{},
+		expErr:  ErrSSEEncryptedObject,
+	},
+	{
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
+		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
+		expErr:  ErrInvalidEncryptionParameters,
+	},
+	{
+		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
+		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
+		expErr:  ErrObjectTampered,
+	},
+}
+
+func TestDecryptObjectInfo(t *testing.T) {
+	for i, test := range decryptObjectInfoTests {
+		if err, encrypted := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
+			t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
+		} else if enc := test.info.IsEncrypted(); encrypted && enc != encrypted {
+			t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)
+		} else if !encrypted && enc != encrypted {
+			t.Errorf("Test %d: Decryption thinks object is not encrypted but it is", i)
+		}
+	}
+}

cmd/endpoint.go

@@ -22,6 +22,7 @@ import (
 	"net/url"
 	"path"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -64,18 +65,9 @@ func (endpoint Endpoint) Type() EndpointType {
 	return URLEndpointType
 }
 
-// SetHTTPS - sets secure http for URLEndpointType.
-func (endpoint Endpoint) SetHTTPS() {
-	if endpoint.Host != "" {
-		endpoint.Scheme = "https"
-	}
-}
-
-// SetHTTP - sets insecure http for URLEndpointType.
-func (endpoint Endpoint) SetHTTP() {
-	if endpoint.Host != "" {
-		endpoint.Scheme = "http"
-	}
+// IsHTTPS - returns true if secure for URLEndpointType.
+func (endpoint Endpoint) IsHTTPS() bool {
+	return endpoint.Scheme == "https"
 }
 
 // NewEndpoint - returns new endpoint based on given arguments.
@@ -130,6 +122,26 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			return ep, fmt.Errorf("empty or root path is not supported in URL endpoint")
 		}
 
+		// On windows having a preceding "/" will cause problems, if the
+		// command line already has C:/<export-folder/ in it. Final resulting
+		// path on windows might become C:/C:/ this will cause problems
+		// of starting minio server properly in distributed mode on windows.
+		// As a special case make sure to trim the separator.
+
+		// NOTE: It is also perfectly fine for windows users to have a path
+		// without C:/ since at that point we treat it as relative path
+		// and obtain the full filesystem path as well. Providing C:/
+		// style is necessary to provide paths other than C:/,
+		// such as F:/, D:/ etc.
+		//
+		// Another additional benefit here is that this style also
+		// supports providing \\host\share support as well.
+		if runtime.GOOS == globalWindowsOSName {
+			if filepath.VolumeName(u.Path[1:]) != "" {
+				u.Path = u.Path[1:]
+			}
+		}
+
 		isLocal, err = isLocalHost(host)
 		if err != nil {
 			return ep, err
@@ -169,18 +181,9 @@ func (endpoints EndpointList) Less(i, j int) bool {
 	return endpoints[i].String() < endpoints[j].String()
 }
 
-// SetHTTPS - sets secure http for URLEndpointType.
-func (endpoints EndpointList) SetHTTPS() {
-	for i := range endpoints {
-		endpoints[i].SetHTTPS()
-	}
-}
-
-// SetHTTP - sets insecure http for URLEndpointType.
-func (endpoints EndpointList) SetHTTP() {
-	for i := range endpoints {
-		endpoints[i].SetHTTP()
-	}
+// IsHTTPS - returns true if secure for URLEndpointType.
+func (endpoints EndpointList) IsHTTPS() bool {
+	return endpoints[0].IsHTTPS()
 }
 
 // NewEndpointList - returns new endpoint list based on input args.

cmd/erasure-createfile.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"hash"
 	"io"
+
+	"github.com/minio/minio/pkg/errors"
 )
 
 // CreateFile creates a new bitrot encoded file spread over all available disks. CreateFile will create
@@ -26,14 +28,14 @@ import (
 // be used to protect the erasure encoded file.
 func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) {
 	if !algorithm.Available() {
-		return f, traceError(errBitrotHashAlgoInvalid)
+		return f, errors.Trace(errBitrotHashAlgoInvalid)
 	}
 	f.Checksums = make([][]byte, len(s.disks))
 	hashers := make([]hash.Hash, len(s.disks))
 	for i := range hashers {
 		hashers[i] = algorithm.New()
 	}
-	errChans, errors := make([]chan error, len(s.disks)), make([]error, len(s.disks))
+	errChans, errs := make([]chan error, len(s.disks)), make([]error, len(s.disks))
 	for i := range errChans {
 		errChans[i] = make(chan error, 1) // create buffered channel to let finished go-routines die early
 	}
@@ -53,19 +55,19 @@ func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer [
 				return f, err
 			}
 		} else {
-			return f, traceError(err)
+			return f, errors.Trace(err)
 		}
 
 		for i := range errChans { // span workers
 			go erasureAppendFile(s.disks[i], volume, path, hashers[i], blocks[i], errChans[i])
 		}
 		for i := range errChans { // what until all workers are finished
-			errors[i] = <-errChans[i]
+			errs[i] = <-errChans[i]
 		}
-		if err = reduceWriteQuorumErrs(errors, objectOpIgnoredErrs, writeQuorum); err != nil {
+		if err = reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum); err != nil {
 			return f, err
 		}
-		s.disks = evalDisks(s.disks, errors)
+		s.disks = evalDisks(s.disks, errs)
 		f.Size += int64(n)
 	}
 
@@ -83,7 +85,7 @@ func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer [
 // the hash of the written data. It sends the write error (or nil) over the error channel.
 func erasureAppendFile(disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) {
 	if disk == OfflineDisk {
-		errChan <- traceError(errDiskNotFound)
+		errChan <- errors.Trace(errDiskNotFound)
 		return
 	}
 	err := disk.AppendFile(volume, path, buf)

cmd/erasure-createfile_test.go

@@ -116,3 +116,77 @@ func TestErasureCreateFile(t *testing.T) {
 		setup.Remove()
 	}
 }
+
+// Benchmarks
+
+func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
+	setup, err := newErasureTestSetup(data, parity, blockSizeV1)
+	if err != nil {
+		b.Fatalf("failed to create test setup: %v", err)
+	}
+	defer setup.Remove()
+	storage, err := NewErasureStorage(setup.disks, data, parity)
+	if err != nil {
+		b.Fatalf("failed to create ErasureStorage: %v", err)
+	}
+	buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
+	content := make([]byte, size)
+
+	for i := 0; i < dataDown; i++ {
+		storage.disks[i] = OfflineDisk
+	}
+	for i := data; i < data+parityDown; i++ {
+		storage.disks[i] = OfflineDisk
+	}
+
+	b.ResetTimer()
+	b.SetBytes(size)
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		_, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func BenchmarkErasureWriteQuick(b *testing.B) {
+	const size = 12 * 1024 * 1024
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
+}
+
+func BenchmarkErasureWrite_4_64KB(b *testing.B) {
+	const size = 64 * 1024
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
+}
+
+func BenchmarkErasureWrite_8_20MB(b *testing.B) {
+	const size = 20 * 1024 * 1024
+	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 0, size, b) })
+	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 1, size, b) })
+	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 1, 0, size, b) })
+	b.Run(" 0000|XXX0 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 3, size, b) })
+	b.Run(" XXX0|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 3, 0, size, b) })
+}
+
+func BenchmarkErasureWrite_12_30MB(b *testing.B) {
+	const size = 30 * 1024 * 1024
+	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 0, size, b) })
+	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 1, size, b) })
+	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 1, 0, size, b) })
+	b.Run(" 000000|XXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 5, size, b) })
+	b.Run(" XXXXX0|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 5, 0, size, b) })
+}
+
+func BenchmarkErasureWrite_16_40MB(b *testing.B) {
+	const size = 40 * 1024 * 1024
+	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 0, size, b) })
+	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 1, size, b) })
+	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 1, 0, size, b) })
+	b.Run(" 00000000|XXXXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 7, size, b) })
+	b.Run(" XXXXXXX0|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 7, 0, size, b) })
+}

cmd/erasure-healfile.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"hash"
 	"strings"
+
+	"github.com/minio/minio/pkg/errors"
 )
 
 // HealFile tries to reconstruct an erasure-coded file spread over all
@@ -48,7 +50,7 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
 	f ErasureFileInfo, err error) {
 
 	if !alg.Available() {
-		return f, traceError(errBitrotHashAlgoInvalid)
+		return f, errors.Trace(errBitrotHashAlgoInvalid)
 	}
 
 	// Initialization
@@ -76,12 +78,14 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
 		blocks[i] = make([]byte, chunksize)
 	}
 	var chunkOffset, blockOffset int64
-	for ; blockOffset < size; blockOffset += blocksize {
+
+	// The for loop below is entered when size == 0 and
+	// blockOffset == 0 to allow for reconstructing empty files.
+	for ; blockOffset == 0 || blockOffset < size; blockOffset += blocksize {
 		// last iteration may have less than blocksize data
 		// left, so chunksize needs to be recomputed.
 		if size < blockOffset+blocksize {
-			blocksize = size - blockOffset
-			chunksize = getChunkSize(blocksize, s.dataBlocks)
+			chunksize = getChunkSize(size-blockOffset, s.dataBlocks)
 			for i := range blocks {
 				blocks[i] = blocks[i][:chunksize]
 			}
@@ -119,9 +123,13 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
 		// iteration
 		chunkOffset += chunksize
 
-		// reconstruct data - this computes all data and parity shards
-		if err = s.ErasureDecodeDataAndParityBlocks(blocks); err != nil {
-			return f, err
+		// reconstruct data - this computes all data and
+		// parity shards - but we skip this step if we are
+		// reconstructing an empty file.
+		if chunksize > 0 {
+			if err = s.ErasureDecodeDataAndParityBlocks(blocks); err != nil {
+				return f, err
+			}
 		}
 
 		// write computed shards as chunks on file in each
@@ -144,7 +152,7 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
 		// If all disks had write errors we quit.
 		if !writeSucceeded {
 			// build error from all write errors
-			return f, traceError(joinWriteErrors(writeErrors))
+			return f, errors.Trace(joinWriteErrors(writeErrors))
 		}
 	}
 

cmd/erasure-readfile.go

@@ -18,6 +18,8 @@ package cmd
 
 import (
 	"io"
+
+	"github.com/minio/minio/pkg/errors"
 )
 
 // ReadFile reads as much data as requested from the file under the given volume and path and writes the data to the provided writer.
@@ -25,13 +27,13 @@ import (
 // up to the given length. If parts of the file are corrupted ReadFile tries to reconstruct the data.
 func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset, length int64, totalLength int64, checksums [][]byte, algorithm BitrotAlgorithm, blocksize int64) (f ErasureFileInfo, err error) {
 	if offset < 0 || length < 0 {
-		return f, traceError(errUnexpected)
+		return f, errors.Trace(errUnexpected)
 	}
 	if offset+length > totalLength {
-		return f, traceError(errUnexpected)
+		return f, errors.Trace(errUnexpected)
 	}
 	if !algorithm.Available() {
-		return f, traceError(errBitrotHashAlgoInvalid)
+		return f, errors.Trace(errBitrotHashAlgoInvalid)
 	}
 
 	f.Checksums = make([][]byte, len(s.disks))
@@ -66,7 +68,7 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset,
 		}
 		err = s.readConcurrent(volume, path, blockOffset, blocks, verifiers, errChans)
 		if err != nil {
-			return f, traceError(errXLReadQuorum)
+			return f, errors.Trace(errXLReadQuorum)
 		}
 
 		writeLength := blocksize - startOffset
@@ -150,7 +152,7 @@ func erasureReadBlocksConcurrent(disks []StorageAPI, volume, path string, offset
 // It sends the returned error through the error channel.
 func erasureReadFromFile(disk StorageAPI, volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier, errChan chan<- error) {
 	if disk == OfflineDisk {
-		errChan <- traceError(errDiskNotFound)
+		errChan <- errors.Trace(errDiskNotFound)
 		return
 	}
 	_, err := disk.ReadFile(volume, path, offset, buffer, verifier)

cmd/erasure-readfile_test.go

@@ -222,3 +222,92 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 		buf.Reset()
 	}
 }
+
+// Benchmarks
+
+func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
+	setup, err := newErasureTestSetup(data, parity, blockSizeV1)
+	if err != nil {
+		b.Fatalf("failed to create test setup: %v", err)
+	}
+	defer setup.Remove()
+	storage, err := NewErasureStorage(setup.disks, data, parity)
+	if err != nil {
+		b.Fatalf("failed to create ErasureStorage: %v", err)
+	}
+
+	content := make([]byte, size)
+	buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
+	file, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
+	if err != nil {
+		b.Fatalf("failed to create erasure test file: %v", err)
+	}
+	checksums := file.Checksums
+
+	for i := 0; i < dataDown; i++ {
+		storage.disks[i] = OfflineDisk
+	}
+	for i := data; i < data+parityDown; i++ {
+		storage.disks[i] = OfflineDisk
+	}
+
+	b.ResetTimer()
+	b.SetBytes(size)
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		if file, err = storage.ReadFile(bytes.NewBuffer(content[:0]), "testbucket", "object", 0, size, size, checksums, DefaultBitrotAlgorithm, blockSizeV1); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func BenchmarkErasureReadQuick(b *testing.B) {
+	const size = 12 * 1024 * 1024
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 0, size, b) })
+	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 1, size, b) })
+}
+
+func BenchmarkErasureRead_4_64KB(b *testing.B) {
+	const size = 64 * 1024
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 0, size, b) })
+	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 1, size, b) })
+	b.Run(" 00|XX ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 2, size, b) })
+	b.Run(" XX|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 2, 0, size, b) })
+}
+
+func BenchmarkErasureRead_8_20MB(b *testing.B) {
+	const size = 20 * 1024 * 1024
+	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 0, size, b) })
+	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 1, size, b) })
+	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 1, 0, size, b) })
+	b.Run(" X000|X000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 1, 1, size, b) })
+	b.Run(" 0000|XXXX ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 4, size, b) })
+	b.Run(" XX00|XX00 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 2, 2, size, b) })
+	b.Run(" XXXX|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 4, 0, size, b) })
+}
+
+func BenchmarkErasureRead_12_30MB(b *testing.B) {
+	const size = 30 * 1024 * 1024
+	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 0, size, b) })
+	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 1, size, b) })
+	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 1, 0, size, b) })
+	b.Run(" X00000|X00000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 1, 1, size, b) })
+	b.Run(" 000000|XXXXXX ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 6, size, b) })
+	b.Run(" XXX000|XXX000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 3, 3, size, b) })
+	b.Run(" XXXXXX|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 6, 0, size, b) })
+}
+
+func BenchmarkErasureRead_16_40MB(b *testing.B) {
+	const size = 40 * 1024 * 1024
+	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 0, size, b) })
+	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 1, size, b) })
+	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 1, 0, size, b) })
+	b.Run(" X0000000|X0000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 1, 1, size, b) })
+	b.Run(" 00000000|XXXXXXXX ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 8, size, b) })
+	b.Run(" XXXX0000|XXXX0000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 4, 4, size, b) })
+	b.Run(" XXXXXXXX|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 8, 0, size, b) })
+}

cmd/erasure-utils.go

@@ -21,6 +21,7 @@ import (
 	"io"
 
 	"github.com/klauspost/reedsolomon"
+	"github.com/minio/minio/pkg/errors"
 )
 
 // getDataBlockLen - get length of data blocks from encoded blocks.
@@ -38,17 +39,17 @@ func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
 func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
 	// Offset and out size cannot be negative.
 	if offset < 0 || length < 0 {
-		return 0, traceError(errUnexpected)
+		return 0, errors.Trace(errUnexpected)
 	}
 
 	// Do we have enough blocks?
 	if len(enBlocks) < dataBlocks {
-		return 0, traceError(reedsolomon.ErrTooFewShards)
+		return 0, errors.Trace(reedsolomon.ErrTooFewShards)
 	}
 
 	// Do we have enough data?
 	if int64(getDataBlockLen(enBlocks, dataBlocks)) < length {
-		return 0, traceError(reedsolomon.ErrShortData)
+		return 0, errors.Trace(reedsolomon.ErrShortData)
 	}
 
 	// Counter to decrement total left to write.
@@ -76,7 +77,7 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
 		if write < int64(len(block)) {
 			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
 			if err != nil {
-				return 0, traceError(err)
+				return 0, errors.Trace(err)
 			}
 			totalWritten += n
 			break
@@ -84,7 +85,7 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
 		// Copy the block.
 		n, err := io.Copy(dst, bytes.NewReader(block))
 		if err != nil {
-			return 0, traceError(err)
+			return 0, errors.Trace(err)
 		}
 
 		// Decrement output size.

cmd/erasure.go

@@ -21,6 +21,7 @@ import (
 	"hash"
 
 	"github.com/klauspost/reedsolomon"
+	"github.com/minio/minio/pkg/errors"
 )
 
 // OfflineDisk represents an unavailable disk.
@@ -46,7 +47,7 @@ type ErasureStorage struct {
 func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int) (s ErasureStorage, err error) {
 	erasure, err := reedsolomon.New(dataBlocks, parityBlocks)
 	if err != nil {
-		return s, traceErrorf("failed to create erasure coding: %v", err)
+		return s, errors.Tracef("failed to create erasure coding: %v", err)
 	}
 	s = ErasureStorage{
 		disks:        make([]StorageAPI, len(disks)),
@@ -63,10 +64,10 @@ func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int) (s Eras
 func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) {
 	encoded, err := s.erasure.Split(data)
 	if err != nil {
-		return nil, traceErrorf("failed to split data: %v", err)
+		return nil, errors.Tracef("failed to split data: %v", err)
 	}
 	if err = s.erasure.Encode(encoded); err != nil {
-		return nil, traceErrorf("failed to encode data: %v", err)
+		return nil, errors.Tracef("failed to encode data: %v", err)
 	}
 	return encoded, nil
 }
@@ -76,7 +77,7 @@ func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) {
 // It returns an error if the decoding failed.
 func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
 	if err := s.erasure.ReconstructData(data); err != nil {
-		return traceErrorf("failed to reconstruct data: %v", err)
+		return errors.Tracef("failed to reconstruct data: %v", err)
 	}
 	return nil
 }
@@ -85,7 +86,7 @@ func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
 // It returns an error if the decoding failed.
 func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(data [][]byte) error {
 	if err := s.erasure.Reconstruct(data); err != nil {
-		return traceErrorf("failed to reconstruct data: %v", err)
+		return errors.Tracef("failed to reconstruct data: %v", err)
 	}
 	return nil
 }

cmd/errors.go

@@ -1,158 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-)
-
-// Holds the current directory path. Used for trimming path in traceError()
-var rootPath string
-
-// Figure out the rootPath
-func initError() {
-	// Root path is automatically determined from the calling function's source file location.
-	// Catch the calling function's source file path.
-	_, file, _, _ := runtime.Caller(1)
-	// Save the directory alone.
-	rootPath = filepath.Dir(file)
-}
-
-// Represents a stack frame in the stack trace.
-type traceInfo struct {
-	file string // File where error occurred
-	line int    // Line where error occurred
-	name string // Name of the function where error occurred
-}
-
-// Error - error type containing cause and the stack trace.
-type Error struct {
-	e     error       // Holds the cause error
-	trace []traceInfo // stack trace
-	errs  []error     // Useful for XL to hold errors from all disks
-}
-
-// Implement error interface.
-func (e Error) Error() string {
-	return e.e.Error()
-}
-
-// Trace - returns stack trace.
-func (e Error) Trace() []string {
-	var traceArr []string
-	for _, info := range e.trace {
-		traceArr = append(traceArr, fmt.Sprintf("%s:%d:%s",
-			info.file, info.line, info.name))
-	}
-	return traceArr
-}
-
-// NewStorageError - return new Error type.
-func traceError(e error, errs ...error) error {
-	if e == nil {
-		return nil
-	}
-	err := &Error{}
-	err.e = e
-	err.errs = errs
-
-	stack := make([]uintptr, 40)
-	length := runtime.Callers(2, stack)
-	if length > len(stack) {
-		length = len(stack)
-	}
-	stack = stack[:length]
-
-	for _, pc := range stack {
-		pc = pc - 1
-		fn := runtime.FuncForPC(pc)
-		file, line := fn.FileLine(pc)
-		name := fn.Name()
-		if hasSuffix(name, "ServeHTTP") {
-			break
-		}
-		if hasSuffix(name, "runtime.") {
-			break
-		}
-
-		file = strings.TrimPrefix(file, rootPath+string(os.PathSeparator))
-		name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
-		err.trace = append(err.trace, traceInfo{file, line, name})
-	}
-
-	return err
-}
-
-// Returns the underlying cause error.
-func errorCause(err error) error {
-	if e, ok := err.(*Error); ok {
-		err = e.e
-	}
-	return err
-}
-
-// Returns slice of underlying cause error.
-func errorsCause(errs []error) []error {
-	cerrs := make([]error, len(errs))
-	for i, err := range errs {
-		if err == nil {
-			continue
-		}
-		cerrs[i] = errorCause(err)
-	}
-	return cerrs
-}
-
-// Collection of basic errors.
-var baseErrs = []error{
-	errDiskNotFound,
-	errFaultyDisk,
-	errFaultyRemoteDisk,
-}
-
-var baseIgnoredErrs = baseErrs
-
-// isErrIgnored returns whether given error is ignored or not.
-func isErrIgnored(err error, ignoredErrs ...error) bool {
-	err = errorCause(err)
-	for _, ignoredErr := range ignoredErrs {
-		if ignoredErr == err {
-			return true
-		}
-	}
-	return false
-}
-
-// isErr returns whether given error is exact error.
-func isErr(err error, errs ...error) bool {
-	err = errorCause(err)
-	for _, exactErr := range errs {
-		if err == exactErr {
-			return true
-		}
-	}
-	return false
-}
-
-// traceErrorf behaves like fmt.traceErrorf but also traces the returned error.
-func traceErrorf(format string, args ...interface{}) error {
-	return traceError(fmt.Errorf(format, args...))
-}

cmd/event-notifier.go

@@ -27,6 +27,7 @@ import (
 	"sync"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
 )
 
@@ -60,7 +61,7 @@ type internalNotifier struct {
 	// Connected listeners is a map of listener ARNs to channels
 	// on which the ListenBucket API handler go routine is waiting
 	// for events to send to a client.
-	connectedListeners map[string]chan []NotificationEvent
+	connectedListeners map[string]*listenChan
 
 	rwMutex *sync.RWMutex
 }
@@ -106,10 +107,10 @@ func newNotificationEvent(event eventData) NotificationEvent {
 	}
 
 	// Fetch the region.
-	region := serverConfig.GetRegion()
+	region := globalServerConfig.GetRegion()
 
 	// Fetch the credentials.
-	creds := serverConfig.GetCredential()
+	creds := globalServerConfig.GetCredential()
 
 	// Time when Minio finished processing the request.
 	eventTime := UTCNow()
@@ -167,13 +168,13 @@ func newNotificationEvent(event eventData) NotificationEvent {
 
 	// For all other events we should set ETag and Size.
 	nEvent.S3.Object = objectMeta{
-		Key:         escapedObj,
-		ETag:        event.ObjInfo.ETag,
-		Size:        event.ObjInfo.Size,
-		ContentType: event.ObjInfo.ContentType,
-		UserDefined: event.ObjInfo.UserDefined,
-		VersionID:   "1",
-		Sequencer:   uniqueID,
+		Key:          escapedObj,
+		ETag:         event.ObjInfo.ETag,
+		Size:         event.ObjInfo.Size,
+		ContentType:  event.ObjInfo.ContentType,
+		UserMetadata: event.ObjInfo.UserDefined,
+		VersionID:    "1",
+		Sequencer:    uniqueID,
 	}
 
 	// Success.
@@ -206,7 +207,7 @@ func (en eventNotifier) GetInternalTarget(arn string) *listenerLogger {
 }
 
 // Set a new sns target for an input sns ARN.
-func (en *eventNotifier) AddListenerChan(snsARN string, listenerCh chan []NotificationEvent) error {
+func (en *eventNotifier) AddListenerChan(snsARN string, listenerCh *listenChan) error {
 	if listenerCh == nil {
 		return errInvalidArgument
 	}
@@ -229,9 +230,9 @@ func (en *eventNotifier) SendListenerEvent(arn string, event []NotificationEvent
 	en.internal.rwMutex.Lock()
 	defer en.internal.rwMutex.Unlock()
 
-	ch, ok := en.internal.connectedListeners[arn]
+	listenChan, ok := en.internal.connectedListeners[arn]
 	if ok {
-		ch <- event
+		listenChan.sendNotificationEvent(event)
 	}
 	// If the channel is not present we ignore the event.
 	return nil
@@ -378,7 +379,7 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
 		// 'errNoSuchNotifications'.  This is default when no
 		// bucket notifications are found on the bucket.
 		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, traceError(errNoSuchNotifications)
+			return nil, errors.Trace(errNoSuchNotifications)
 		}
 		errorIf(err, "Unable to load bucket-notification for bucket %s", bucket)
 		// Returns error for other errors.
@@ -387,7 +388,7 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
 
 	// if `notifications.xml` is empty we should return NoSuchNotifications.
 	if buffer.Len() == 0 {
-		return nil, traceError(errNoSuchNotifications)
+		return nil, errors.Trace(errNoSuchNotifications)
 	}
 
 	// Unmarshal notification bytes.
@@ -395,7 +396,7 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
 	notificationCfg := &notificationConfig{}
 	// Unmarshal notification bytes only if we read data.
 	if err = xml.Unmarshal(notificationConfigBytes, notificationCfg); err != nil {
-		return nil, traceError(err)
+		return nil, errors.Trace(err)
 	}
 
 	// Return success.
@@ -429,7 +430,7 @@ func loadListenerConfig(bucket string, objAPI ObjectLayer) ([]listenerConfig, er
 		// 'errNoSuchNotifications'.  This is default when no
 		// bucket listeners are found on the bucket
 		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, traceError(errNoSuchNotifications)
+			return nil, errors.Trace(errNoSuchNotifications)
 		}
 		errorIf(err, "Unable to load bucket-listeners for bucket %s", bucket)
 		// Returns error for other errors.
@@ -438,14 +439,14 @@ func loadListenerConfig(bucket string, objAPI ObjectLayer) ([]listenerConfig, er
 
 	// if `listener.json` is empty we should return NoSuchNotifications.
 	if buffer.Len() == 0 {
-		return nil, traceError(errNoSuchNotifications)
+		return nil, errors.Trace(errNoSuchNotifications)
 	}
 
 	var lCfg []listenerConfig
 	lConfigBytes := buffer.Bytes()
 	if err = json.Unmarshal(lConfigBytes, &lCfg); err != nil {
 		errorIf(err, "Unable to unmarshal listener config from JSON.")
-		return nil, traceError(err)
+		return nil, errors.Trace(err)
 	}
 
 	// Return success.
@@ -552,13 +553,13 @@ func removeListenerConfig(bucket string, objAPI ObjectLayer) error {
 func loadNotificationAndListenerConfig(bucketName string, objAPI ObjectLayer) (nCfg *notificationConfig, lCfg []listenerConfig, err error) {
 	// Loads notification config if any.
 	nCfg, err = loadNotificationConfig(bucketName, objAPI)
-	if err != nil && !isErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
+	if err != nil && !errors.IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
 		return nil, nil, err
 	}
 
 	// Loads listener config if any.
 	lCfg, err = loadListenerConfig(bucketName, objAPI)
-	if err != nil && !isErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
+	if err != nil && !errors.IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
 		return nil, nil, err
 	}
 	return nCfg, lCfg, nil
@@ -595,7 +596,7 @@ func addQueueTarget(queueTargets map[string]*logrus.Logger,
 	newTargetFunc func(string) (*logrus.Logger, error)) (string, error) {
 
 	// Construct the queue ARN for AMQP.
-	queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueType
+	queueARN := minioSqs + globalServerConfig.GetRegion() + ":" + accountID + ":" + queueType
 
 	// Queue target if already initialized we move to the next ARN.
 	if _, ok := queueTargets[queueARN]; ok {
@@ -618,7 +619,7 @@ func addQueueTarget(queueTargets map[string]*logrus.Logger,
 func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	queueTargets := make(map[string]*logrus.Logger)
 	// Load all amqp targets, initialize their respective loggers.
-	for accountID, amqpN := range serverConfig.Notify.GetAMQP() {
+	for accountID, amqpN := range globalServerConfig.Notify.GetAMQP() {
 		if !amqpN.Enable {
 			continue
 		}
@@ -637,7 +638,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load all mqtt targets, initialize their respective loggers.
-	for accountID, mqttN := range serverConfig.Notify.GetMQTT() {
+	for accountID, mqttN := range globalServerConfig.Notify.GetMQTT() {
 		if !mqttN.Enable {
 			continue
 		}
@@ -656,7 +657,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load all nats targets, initialize their respective loggers.
-	for accountID, natsN := range serverConfig.Notify.GetNATS() {
+	for accountID, natsN := range globalServerConfig.Notify.GetNATS() {
 		if !natsN.Enable {
 			continue
 		}
@@ -675,7 +676,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load redis targets, initialize their respective loggers.
-	for accountID, redisN := range serverConfig.Notify.GetRedis() {
+	for accountID, redisN := range globalServerConfig.Notify.GetRedis() {
 		if !redisN.Enable {
 			continue
 		}
@@ -694,7 +695,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load Webhook targets, initialize their respective loggers.
-	for accountID, webhookN := range serverConfig.Notify.GetWebhook() {
+	for accountID, webhookN := range globalServerConfig.Notify.GetWebhook() {
 		if !webhookN.Enable {
 			continue
 		}
@@ -704,7 +705,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load elastic targets, initialize their respective loggers.
-	for accountID, elasticN := range serverConfig.Notify.GetElasticSearch() {
+	for accountID, elasticN := range globalServerConfig.Notify.GetElasticSearch() {
 		if !elasticN.Enable {
 			continue
 		}
@@ -723,7 +724,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load PostgreSQL targets, initialize their respective loggers.
-	for accountID, pgN := range serverConfig.Notify.GetPostgreSQL() {
+	for accountID, pgN := range globalServerConfig.Notify.GetPostgreSQL() {
 		if !pgN.Enable {
 			continue
 		}
@@ -742,7 +743,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load MySQL targets, initialize their respective loggers.
-	for accountID, msqlN := range serverConfig.Notify.GetMySQL() {
+	for accountID, msqlN := range globalServerConfig.Notify.GetMySQL() {
 		if !msqlN.Enable {
 			continue
 		}
@@ -761,7 +762,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load Kafka targets, initialize their respective loggers.
-	for accountID, kafkaN := range serverConfig.Notify.GetKafka() {
+	for accountID, kafkaN := range globalServerConfig.Notify.GetKafka() {
 		if !kafkaN.Enable {
 			continue
 		}
@@ -833,7 +834,7 @@ func initEventNotifier(objAPI ObjectLayer) error {
 			rwMutex:            &sync.RWMutex{},
 			targets:            listenTargets,
 			listenerConfigs:    lConfigs,
-			connectedListeners: make(map[string]chan []NotificationEvent),
+			connectedListeners: make(map[string]*listenChan),
 		},
 	}
 

cmd/event-notifier_test.go

@@ -23,6 +23,8 @@ import (
 	"reflect"
 	"testing"
 	"time"
+
+	"github.com/minio/minio/pkg/errors"
 )
 
 // Test InitEventNotifier with faulty disks
@@ -71,7 +73,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
 	}
 	// Test initEventNotifier() with faulty disks
 	for i := 1; i <= 3; i++ {
-		if err := initEventNotifier(xl); errorCause(err) != errFaultyDisk {
+		if err := initEventNotifier(xl); errors.Cause(err) != errFaultyDisk {
 			t.Fatal("Unexpected error:", err)
 		}
 	}
@@ -98,7 +100,7 @@ func TestInitEventNotifierWithPostgreSQL(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	serverConfig.Notify.SetPostgreSQLByID("1", postgreSQLNotify{Enable: true})
+	globalServerConfig.Notify.SetPostgreSQLByID("1", postgreSQLNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("PostgreSQL config didn't fail.")
 	}
@@ -125,7 +127,7 @@ func TestInitEventNotifierWithNATS(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	serverConfig.Notify.SetNATSByID("1", natsNotify{Enable: true})
+	globalServerConfig.Notify.SetNATSByID("1", natsNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("NATS config didn't fail.")
 	}
@@ -152,7 +154,7 @@ func TestInitEventNotifierWithWebHook(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	serverConfig.Notify.SetWebhookByID("1", webhookNotify{Enable: true})
+	globalServerConfig.Notify.SetWebhookByID("1", webhookNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("WebHook config didn't fail.")
 	}
@@ -179,7 +181,7 @@ func TestInitEventNotifierWithAMQP(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	serverConfig.Notify.SetAMQPByID("1", amqpNotify{Enable: true})
+	globalServerConfig.Notify.SetAMQPByID("1", amqpNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("AMQP config didn't fail.")
 	}
@@ -206,7 +208,7 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	serverConfig.Notify.SetElasticSearchByID("1", elasticSearchNotify{Enable: true})
+	globalServerConfig.Notify.SetElasticSearchByID("1", elasticSearchNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("ElasticSearch config didn't fail.")
 	}
@@ -233,7 +235,7 @@ func TestInitEventNotifierWithRedis(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	serverConfig.Notify.SetRedisByID("1", redisNotify{Enable: true})
+	globalServerConfig.Notify.SetRedisByID("1", redisNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("Redis config didn't fail.")
 	}
@@ -415,7 +417,7 @@ func TestListenBucketNotification(t *testing.T) {
 
 	listenARN := fmt.Sprintf("%s:%s:1:%s-%s",
 		minioTopic,
-		serverConfig.GetRegion(),
+		globalServerConfig.GetRegion(),
 		snsTypeMinio,
 		s.testServer.Server.Listener.Addr(),
 	)
@@ -462,11 +464,9 @@ func TestListenBucketNotification(t *testing.T) {
 	}
 
 	// Create a new notification event channel.
-	nEventCh := make(chan []NotificationEvent)
-	// Close the listener channel.
-	defer close(nEventCh)
+	nListenCh := newListenChan()
 	// Add events channel for listener.
-	if err := globalEventNotifier.AddListenerChan(listenARN, nEventCh); err != nil {
+	if err := globalEventNotifier.AddListenerChan(listenARN, nListenCh); err != nil {
 		t.Fatalf("Test Setup error: %v", err)
 	}
 	// Remove listen channel after the writer has closed or the
@@ -489,7 +489,7 @@ func TestListenBucketNotification(t *testing.T) {
 	// Wait for the event notification here, if nothing is received within 30 seconds,
 	// test error will be fired
 	select {
-	case n := <-nEventCh:
+	case n := <-nListenCh.dataCh:
 		// Check that received event
 		if len(n) == 0 {
 			t.Fatal("Unexpected error occurred")
@@ -497,9 +497,7 @@ func TestListenBucketNotification(t *testing.T) {
 		if n[0].S3.Object.Key != objectName {
 			t.Fatalf("Received wrong object name in notification, expected %s, received %s", n[0].S3.Object.Key, objectName)
 		}
-		break
 	case <-time.After(3 * time.Second):
-		break
 	}
 
 }
@@ -527,7 +525,7 @@ func TestAddRemoveBucketListenerConfig(t *testing.T) {
 	accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
 	accountARN := fmt.Sprintf(
 		"arn:minio:sqs:%s:%s:listen-%s",
-		serverConfig.GetRegion(),
+		globalServerConfig.GetRegion(),
 		accountID,
 		globalMinioAddr,
 	)

cmd/format-config-v1.go

@@ -25,6 +25,7 @@ import (
 	"reflect"
 	"sync"
 
+	errors2 "github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -123,10 +124,10 @@ func (f *formatConfigV1) CheckFS() error {
 // if reading format.json fails with io.EOF.
 func (f *formatConfigV1) LoadFormat(lk *lock.LockedFile) error {
 	_, err := f.ReadFrom(lk)
-	if errorCause(err) == io.EOF {
+	if errors2.Cause(err) == io.EOF {
 		// No data on disk `format.json` still empty
 		// treat it as unformatted disk.
-		return traceError(errUnformattedDisk)
+		return errors2.Trace(errUnformattedDisk)
 	}
 	return err
 }
@@ -136,14 +137,14 @@ func (f *formatConfigV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
 	var fbytes []byte
 	fbytes, err = json.Marshal(f)
 	if err != nil {
-		return 0, traceError(err)
+		return 0, errors2.Trace(err)
 	}
 	if err = lk.Truncate(0); err != nil {
-		return 0, traceError(err)
+		return 0, errors2.Trace(err)
 	}
 	_, err = lk.Write(fbytes)
 	if err != nil {
-		return 0, traceError(err)
+		return 0, errors2.Trace(err)
 	}
 	return int64(len(fbytes)), nil
 }
@@ -152,18 +153,18 @@ func (f *formatConfigV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
 	var fbytes []byte
 	fi, err := lk.Stat()
 	if err != nil {
-		return 0, traceError(err)
+		return 0, errors2.Trace(err)
 	}
 	fbytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
 	if err != nil {
-		return 0, traceError(err)
+		return 0, errors2.Trace(err)
 	}
 	if len(fbytes) == 0 {
-		return 0, traceError(io.EOF)
+		return 0, errors2.Trace(io.EOF)
 	}
 	// Decode `format.json`.
 	if err = json.Unmarshal(fbytes, f); err != nil {
-		return 0, traceError(err)
+		return 0, errors2.Trace(err)
 	}
 	return int64(len(fbytes)), nil
 }

cmd/format-config-v1_test.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	errors2 "github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 )
@@ -316,7 +317,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 // a given disk to test healing a corrupted disk
 func TestFormatXLHealCorruptedDisks(t *testing.T) {
 	// Create an instance of xl backend.
-	obj, fsDirs, err := prepareXL()
+	obj, fsDirs, err := prepareXL16()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -390,7 +391,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 // some of format.json
 func TestFormatXLReorderByInspection(t *testing.T) {
 	// Create an instance of xl backend.
-	obj, fsDirs, err := prepareXL()
+	obj, fsDirs, err := prepareXL16()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -756,7 +757,7 @@ func TestFSCheckFormatFSErr(t *testing.T) {
 			t.Errorf("Test %d: Should fail with expected %s, got nil", i+1, testCase.formatCheckErr)
 		}
 		if err != nil && !testCase.shouldPass {
-			if errorCause(err).Error() != testCase.formatCheckErr.Error() {
+			if errors2.Cause(err).Error() != testCase.formatCheckErr.Error() {
 				t.Errorf("Test %d: Should fail with expected %s, got %s", i+1, testCase.formatCheckErr, err)
 			}
 		}

cmd/fs-v1-helpers.go

@@ -22,6 +22,8 @@ import (
 	"os"
 	pathutil "path"
 	"runtime"
+
+	"github.com/minio/minio/pkg/errors"
 )
 
 // Removes only the file at given path does not remove
@@ -29,11 +31,11 @@ import (
 // windows automatically.
 func fsRemoveFile(filePath string) (err error) {
 	if filePath == "" {
-		return traceError(errInvalidArgument)
+		return errors.Trace(errInvalidArgument)
 	}
 
 	if err = checkPathLength(filePath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	if err = os.Remove((filePath)); err != nil {
@@ -47,20 +49,20 @@ func fsRemoveFile(filePath string) (err error) {
 // long paths for windows automatically.
 func fsRemoveAll(dirPath string) (err error) {
 	if dirPath == "" {
-		return traceError(errInvalidArgument)
+		return errors.Trace(errInvalidArgument)
 	}
 
 	if err = checkPathLength(dirPath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	if err = os.RemoveAll(dirPath); err != nil {
 		if os.IsPermission(err) {
-			return traceError(errVolumeAccessDenied)
+			return errors.Trace(errVolumeAccessDenied)
 		} else if isSysErrNotEmpty(err) {
-			return traceError(errVolumeNotEmpty)
+			return errors.Trace(errVolumeNotEmpty)
 		}
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	return nil
@@ -70,20 +72,20 @@ func fsRemoveAll(dirPath string) (err error) {
 // paths for windows automatically.
 func fsRemoveDir(dirPath string) (err error) {
 	if dirPath == "" {
-		return traceError(errInvalidArgument)
+		return errors.Trace(errInvalidArgument)
 	}
 
 	if err = checkPathLength(dirPath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	if err = os.Remove((dirPath)); err != nil {
 		if os.IsNotExist(err) {
-			return traceError(errVolumeNotFound)
+			return errors.Trace(errVolumeNotFound)
 		} else if isSysErrNotEmpty(err) {
-			return traceError(errVolumeNotEmpty)
+			return errors.Trace(errVolumeNotEmpty)
 		}
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	return nil
@@ -93,15 +95,15 @@ func fsRemoveDir(dirPath string) (err error) {
 // if it doesn't exist.
 func fsMkdirAll(dirPath string) (err error) {
 	if dirPath == "" {
-		return traceError(errInvalidArgument)
+		return errors.Trace(errInvalidArgument)
 	}
 
 	if err = checkPathLength(dirPath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	if err = os.MkdirAll(dirPath, 0777); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	return nil
@@ -113,27 +115,27 @@ func fsMkdirAll(dirPath string) (err error) {
 // are handled automatically.
 func fsMkdir(dirPath string) (err error) {
 	if dirPath == "" {
-		return traceError(errInvalidArgument)
+		return errors.Trace(errInvalidArgument)
 	}
 
 	if err = checkPathLength(dirPath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	if err = os.Mkdir((dirPath), 0777); err != nil {
 		if os.IsExist(err) {
-			return traceError(errVolumeExists)
+			return errors.Trace(errVolumeExists)
 		} else if os.IsPermission(err) {
-			return traceError(errDiskAccessDenied)
+			return errors.Trace(errDiskAccessDenied)
 		} else if isSysErrNotDir(err) {
 			// File path cannot be verified since
 			// one of the parents is a file.
-			return traceError(errDiskAccessDenied)
+			return errors.Trace(errDiskAccessDenied)
 		} else if isSysErrPathNotFound(err) {
 			// Add specific case for windows.
-			return traceError(errDiskAccessDenied)
+			return errors.Trace(errDiskAccessDenied)
 		}
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	return nil
@@ -146,14 +148,14 @@ func fsMkdir(dirPath string) (err error) {
 // fsStatFileDir, fsStatFile, fsStatDir.
 func fsStat(statLoc string) (os.FileInfo, error) {
 	if statLoc == "" {
-		return nil, traceError(errInvalidArgument)
+		return nil, errors.Trace(errInvalidArgument)
 	}
 	if err := checkPathLength(statLoc); err != nil {
-		return nil, traceError(err)
+		return nil, errors.Trace(err)
 	}
 	fi, err := os.Stat((statLoc))
 	if err != nil {
-		return nil, traceError(err)
+		return nil, errors.Trace(err)
 	}
 
 	return fi, nil
@@ -163,17 +165,17 @@ func fsStat(statLoc string) (os.FileInfo, error) {
 func fsStatVolume(volume string) (os.FileInfo, error) {
 	fi, err := fsStat(volume)
 	if err != nil {
-		err = errorCause(err)
+		err = errors.Cause(err)
 		if os.IsNotExist(err) {
-			return nil, traceError(errVolumeNotFound)
+			return nil, errors.Trace(errVolumeNotFound)
 		} else if os.IsPermission(err) {
-			return nil, traceError(errVolumeAccessDenied)
+			return nil, errors.Trace(errVolumeAccessDenied)
 		}
-		return nil, traceError(err)
+		return nil, errors.Trace(err)
 	}
 
 	if !fi.IsDir() {
-		return nil, traceError(errVolumeAccessDenied)
+		return nil, errors.Trace(errVolumeAccessDenied)
 	}
 
 	return fi, nil
@@ -187,18 +189,18 @@ func osErrToFSFileErr(err error) error {
 	if err == nil {
 		return nil
 	}
-	err = errorCause(err)
+	err = errors.Cause(err)
 	if os.IsNotExist(err) {
-		return traceError(errFileNotFound)
+		return errors.Trace(errFileNotFound)
 	}
 	if os.IsPermission(err) {
-		return traceError(errFileAccessDenied)
+		return errors.Trace(errFileAccessDenied)
 	}
 	if isSysErrNotDir(err) {
-		return traceError(errFileAccessDenied)
+		return errors.Trace(errFileAccessDenied)
 	}
 	if isSysErrPathNotFound(err) {
-		return traceError(errFileNotFound)
+		return errors.Trace(errFileNotFound)
 	}
 	return err
 }
@@ -210,7 +212,7 @@ func fsStatDir(statDir string) (os.FileInfo, error) {
 		return nil, osErrToFSFileErr(err)
 	}
 	if !fi.IsDir() {
-		return nil, traceError(errFileAccessDenied)
+		return nil, errors.Trace(errFileAccessDenied)
 	}
 	return fi, nil
 }
@@ -222,7 +224,7 @@ func fsStatFile(statFile string) (os.FileInfo, error) {
 		return nil, osErrToFSFileErr(err)
 	}
 	if fi.IsDir() {
-		return nil, traceError(errFileAccessDenied)
+		return nil, errors.Trace(errFileAccessDenied)
 	}
 	return fi, nil
 }
@@ -231,10 +233,10 @@ func fsStatFile(statFile string) (os.FileInfo, error) {
 // a readable stream and the size of the readable stream.
 func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
 	if readPath == "" || offset < 0 {
-		return nil, 0, traceError(errInvalidArgument)
+		return nil, 0, errors.Trace(errInvalidArgument)
 	}
 	if err := checkPathLength(readPath); err != nil {
-		return nil, 0, traceError(err)
+		return nil, 0, errors.Trace(err)
 	}
 
 	fr, err := os.Open((readPath))
@@ -245,19 +247,19 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
 	// Stat to get the size of the file at path.
 	st, err := os.Stat((readPath))
 	if err != nil {
-		return nil, 0, traceError(err)
+		return nil, 0, errors.Trace(err)
 	}
 
 	// Verify if its not a regular file, since subsequent Seek is undefined.
 	if !st.Mode().IsRegular() {
-		return nil, 0, traceError(errIsNotRegular)
+		return nil, 0, errors.Trace(errIsNotRegular)
 	}
 
 	// Seek to the requested offset.
 	if offset > 0 {
 		_, err = fr.Seek(offset, os.SEEK_SET)
 		if err != nil {
-			return nil, 0, traceError(err)
+			return nil, 0, errors.Trace(err)
 		}
 	}
 
@@ -268,19 +270,19 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
 // Creates a file and copies data from incoming reader. Staging buffer is used by io.CopyBuffer.
 func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int64) (int64, error) {
 	if filePath == "" || reader == nil {
-		return 0, traceError(errInvalidArgument)
+		return 0, errors.Trace(errInvalidArgument)
 	}
 
 	if err := checkPathLength(filePath); err != nil {
-		return 0, traceError(err)
+		return 0, errors.Trace(err)
 	}
 
 	if err := os.MkdirAll(pathutil.Dir(filePath), 0777); err != nil {
-		return 0, traceError(err)
+		return 0, errors.Trace(err)
 	}
 
 	if err := checkDiskFree(pathutil.Dir(filePath), fallocSize); err != nil {
-		return 0, traceError(err)
+		return 0, errors.Trace(err)
 	}
 
 	writer, err := os.OpenFile((filePath), os.O_CREATE|os.O_WRONLY, 0666)
@@ -292,7 +294,7 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
 	// Fallocate only if the size is final object is known.
 	if fallocSize > 0 {
 		if err = fsFAllocate(int(writer.Fd()), 0, fallocSize); err != nil {
-			return 0, traceError(err)
+			return 0, errors.Trace(err)
 		}
 	}
 
@@ -300,12 +302,12 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
 	if buf != nil {
 		bytesWritten, err = io.CopyBuffer(writer, reader, buf)
 		if err != nil {
-			return 0, traceError(err)
+			return 0, errors.Trace(err)
 		}
 	} else {
 		bytesWritten, err = io.Copy(writer, reader)
 		if err != nil {
-			return 0, traceError(err)
+			return 0, errors.Trace(err)
 		}
 	}
 	return bytesWritten, nil
@@ -314,26 +316,26 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
 // Removes uploadID at destination path.
 func fsRemoveUploadIDPath(basePath, uploadIDPath string) error {
 	if basePath == "" || uploadIDPath == "" {
-		return traceError(errInvalidArgument)
+		return errors.Trace(errInvalidArgument)
 	}
 	if err := checkPathLength(basePath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 	if err := checkPathLength(uploadIDPath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	// List all the entries in uploadID.
 	entries, err := readDir(uploadIDPath)
 	if err != nil && err != errFileNotFound {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	// Delete all the entries obtained from previous readdir.
 	for _, entryPath := range entries {
 		err = fsDeleteFile(basePath, pathJoin(uploadIDPath, entryPath))
 		if err != nil && err != errFileNotFound {
-			return traceError(err)
+			return errors.Trace(err)
 		}
 	}
 
@@ -367,23 +369,23 @@ func fsFAllocate(fd int, offset int64, len int64) (err error) {
 // missing parents if they don't exist.
 func fsRenameFile(sourcePath, destPath string) error {
 	if err := checkPathLength(sourcePath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 	if err := checkPathLength(destPath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 	// Verify if source path exists.
 	if _, err := os.Stat((sourcePath)); err != nil {
 		return osErrToFSFileErr(err)
 	}
 	if err := os.MkdirAll(pathutil.Dir(destPath), 0777); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 	if err := os.Rename((sourcePath), (destPath)); err != nil {
 		if isSysErrCrossDevice(err) {
-			return traceError(fmt.Errorf("%s (%s)->(%s)", errCrossDeviceLink, sourcePath, destPath))
+			return errors.Trace(fmt.Errorf("%s (%s)->(%s)", errCrossDeviceLink, sourcePath, destPath))
 		}
-		return traceError(err)
+		return errors.Trace(err)
 	}
 	return nil
 }
@@ -391,11 +393,11 @@ func fsRenameFile(sourcePath, destPath string) error {
 // fsDeleteFile is a wrapper for deleteFile(), after checking the path length.
 func fsDeleteFile(basePath, deletePath string) error {
 	if err := checkPathLength(basePath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	if err := checkPathLength(deletePath); err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 
 	return deleteFile(basePath, deletePath)

cmd/fs-v1-helpers_test.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"testing"
 
+	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -36,11 +37,11 @@ func TestFSMkdirAll(t *testing.T) {
 	}
 	defer os.RemoveAll(path)
 
-	if err = fsMkdirAll(""); errorCause(err) != errInvalidArgument {
+	if err = fsMkdirAll(""); errors.Cause(err) != errInvalidArgument {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if err = fsMkdirAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
+	if err = fsMkdirAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong {
 		t.Fatal("Unexpected error", err)
 	}
 
@@ -63,13 +64,13 @@ func TestFSRenameFile(t *testing.T) {
 	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil {
 		t.Fatal(err)
 	}
-	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNotFound {
+	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errors.Cause(err) != errFileNotFound {
 		t.Fatal(err)
 	}
-	if err = fsRenameFile(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNameTooLong {
+	if err = fsRenameFile(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errors.Cause(err) != errFileNameTooLong {
 		t.Fatal("Unexpected error", err)
 	}
-	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
+	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong {
 		t.Fatal("Unexpected error", err)
 	}
 }
@@ -84,11 +85,11 @@ func TestFSStats(t *testing.T) {
 
 	// Setup test environment.
 
-	if err = fsMkdir(""); errorCause(err) != errInvalidArgument {
+	if err = fsMkdir(""); errors.Cause(err) != errInvalidArgument {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if err = fsMkdir(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
+	if err = fsMkdir(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong {
 		t.Fatal("Unexpected error", err)
 	}
 
@@ -103,7 +104,7 @@ func TestFSStats(t *testing.T) {
 	// Seek back.
 	reader.Seek(0, 0)
 
-	if err = fsMkdir(pathJoin(path, "success-vol", "success-file")); errorCause(err) != errVolumeExists {
+	if err = fsMkdir(pathJoin(path, "success-vol", "success-file")); errors.Cause(err) != errVolumeExists {
 		t.Fatal("Unexpected error", err)
 	}
 
@@ -191,11 +192,11 @@ func TestFSStats(t *testing.T) {
 	for i, testCase := range testCases {
 		if testCase.srcPath != "" {
 			if _, err := fsStatFile(pathJoin(testCase.srcFSPath, testCase.srcVol,
-				testCase.srcPath)); errorCause(err) != testCase.expectedErr {
+				testCase.srcPath)); errors.Cause(err) != testCase.expectedErr {
 				t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 			}
 		} else {
-			if _, err := fsStatVolume(pathJoin(testCase.srcFSPath, testCase.srcVol)); errorCause(err) != testCase.expectedErr {
+			if _, err := fsStatVolume(pathJoin(testCase.srcFSPath, testCase.srcVol)); errors.Cause(err) != testCase.expectedErr {
 				t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 			}
 		}
@@ -214,11 +215,11 @@ func TestFSCreateAndOpen(t *testing.T) {
 		t.Fatalf("Unable to create directory, %s", err)
 	}
 
-	if _, err = fsCreateFile("", nil, nil, 0); errorCause(err) != errInvalidArgument {
+	if _, err = fsCreateFile("", nil, nil, 0); errors.Cause(err) != errInvalidArgument {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if _, _, err = fsOpenFile("", -1); errorCause(err) != errInvalidArgument {
+	if _, _, err = fsOpenFile("", -1); errors.Cause(err) != errInvalidArgument {
 		t.Fatal("Unexpected error", err)
 	}
 
@@ -252,17 +253,17 @@ func TestFSCreateAndOpen(t *testing.T) {
 
 	for i, testCase := range testCases {
 		_, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0)
-		if errorCause(err) != testCase.expectedErr {
+		if errors.Cause(err) != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
 		_, _, err = fsOpenFile(pathJoin(path, testCase.srcVol, testCase.srcPath), 0)
-		if errorCause(err) != testCase.expectedErr {
+		if errors.Cause(err) != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
 	}
 
 	// Attempt to open a directory.
-	if _, _, err = fsOpenFile(pathJoin(path), 0); errorCause(err) != errIsNotRegular {
+	if _, _, err = fsOpenFile(pathJoin(path), 0); errors.Cause(err) != errIsNotRegular {
 		t.Fatal("Unexpected error", err)
 	}
 }
@@ -364,7 +365,7 @@ func TestFSDeletes(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		if err = fsDeleteFile(testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
+		if err = fsDeleteFile(testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
 	}
@@ -498,11 +499,11 @@ func TestFSRemoves(t *testing.T) {
 
 	for i, testCase := range testCases {
 		if testCase.srcPath != "" {
-			if err = fsRemoveFile(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
+			if err = fsRemoveFile(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr {
 				t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 			}
 		} else {
-			if err = fsRemoveDir(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
+			if err = fsRemoveDir(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr {
 				t.Error(err)
 			}
 		}
@@ -512,11 +513,11 @@ func TestFSRemoves(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err = fsRemoveAll(""); errorCause(err) != errInvalidArgument {
+	if err = fsRemoveAll(""); errors.Cause(err) != errInvalidArgument {
 		t.Fatal(err)
 	}
 
-	if err = fsRemoveAll("my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"); errorCause(err) != errFileNameTooLong {
+	if err = fsRemoveAll("my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"); errors.Cause(err) != errFileNameTooLong {
 		t.Fatal(err)
 	}
 }

cmd/fs-v1-metadata.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/mimedb"
 	"github.com/tidwall/gjson"
@@ -166,15 +167,15 @@ func (m *fsMetaV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
 	var metadataBytes []byte
 	metadataBytes, err = json.Marshal(m)
 	if err != nil {
-		return 0, traceError(err)
+		return 0, errors.Trace(err)
 	}
 
 	if err = lk.Truncate(0); err != nil {
-		return 0, traceError(err)
+		return 0, errors.Trace(err)
 	}
 
 	if _, err = lk.Write(metadataBytes); err != nil {
-		return 0, traceError(err)
+		return 0, errors.Trace(err)
 	}
 
 	// Success.
@@ -222,16 +223,16 @@ func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
 	var fsMetaBuf []byte
 	fi, err := lk.Stat()
 	if err != nil {
-		return 0, traceError(err)
+		return 0, errors.Trace(err)
 	}
 
 	fsMetaBuf, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
 	if err != nil {
-		return 0, traceError(err)
+		return 0, errors.Trace(err)
 	}
 
 	if len(fsMetaBuf) == 0 {
-		return 0, traceError(io.EOF)
+		return 0, errors.Trace(io.EOF)
 	}
 
 	// obtain version.
@@ -243,7 +244,7 @@ func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
 	// Verify if the format is valid, return corrupted format
 	// for unrecognized formats.
 	if !isFSMetaValid(m.Version, m.Format) {
-		return 0, traceError(errCorruptedFormat)
+		return 0, errors.Trace(errCorruptedFormat)
 	}
 
 	// obtain metadata.
@@ -278,9 +279,9 @@ func checkLockedValidFormatFS(fsPath string) (*lock.RLockedFile, error) {
 		if os.IsNotExist(err) {
 			// If format.json not found then
 			// its an unformatted disk.
-			return nil, traceError(errUnformattedDisk)
+			return nil, errors.Trace(errUnformattedDisk)
 		}
-		return nil, traceError(err)
+		return nil, errors.Trace(err)
 	}
 
 	var format = &formatConfigV1{}
@@ -296,7 +297,7 @@ func checkLockedValidFormatFS(fsPath string) (*lock.RLockedFile, error) {
 	}
 
 	//  Always return read lock here and should be closed by the caller.
-	return rlk, traceError(err)
+	return rlk, errors.Trace(err)
 }
 
 // Creates a new format.json if unformatted.
@@ -307,7 +308,7 @@ func createFormatFS(fsPath string) error {
 	// file stored in minioMetaBucket(.minio.sys) directory.
 	lk, err := lock.TryLockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
-		return traceError(err)
+		return errors.Trace(err)
 	}
 	// Close the locked file upon return.
 	defer lk.Close()
@@ -316,7 +317,7 @@ func createFormatFS(fsPath string) error {
 	// writes the new format.json
 	var format = &formatConfigV1{}
 	err = format.LoadFormat(lk)
-	if errorCause(err) == errUnformattedDisk {
+	if errors.Cause(err) == errUnformattedDisk {
 		_, err = newFSFormat().WriteTo(lk)
 		return err
 	}
@@ -338,10 +339,10 @@ func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) {
 			// is blocked if attempted in-turn avoiding corruption on
 			// the backend disk.
 			return rlk, nil
-		case errorCause(err) == errUnformattedDisk:
+		case errors.Cause(err) == errUnformattedDisk:
 			if err = createFormatFS(fsPath); err != nil {
 				// Existing write locks detected.
-				if errorCause(err) == lock.ErrAlreadyLocked {
+				if errors.Cause(err) == lock.ErrAlreadyLocked {
 					// Lock already present, sleep and attempt again.
 					time.Sleep(100 * time.Millisecond)
 					continue
@@ -359,15 +360,15 @@ func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) {
 	}
 }
 
-// Return if the part info in uploadedParts and completeParts are same.
-func isPartsSame(uploadedParts []objectPartInfo, completeParts []completePart) bool {
-	if len(uploadedParts) != len(completeParts) {
+// Return if the part info in uploadedParts and CompleteParts are same.
+func isPartsSame(uploadedParts []objectPartInfo, CompleteParts []CompletePart) bool {
+	if len(uploadedParts) != len(CompleteParts) {
 		return false
 	}
 
-	for i := range completeParts {
-		if uploadedParts[i].Number != completeParts[i].PartNumber ||
-			uploadedParts[i].ETag != completeParts[i].ETag {
+	for i := range CompleteParts {
+		if uploadedParts[i].Number != CompleteParts[i].PartNumber ||
+			uploadedParts[i].ETag != CompleteParts[i].ETag {
 			return false
 		}
 	}

cmd/fs-v1-multipart.go

@@ -25,23 +25,17 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 )
 
-const (
-	// Expiry duration after which the multipart uploads are deemed stale.
-	fsMultipartExpiry = time.Hour * 24 * 14 // 2 weeks.
-	// Cleanup interval when the stale multipart cleanup is initiated.
-	fsMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
-)
-
 // Returns if the prefix is a multipart upload.
 func (fs fsObjects) isMultipartUpload(bucket, prefix string) bool {
 	uploadsIDPath := pathJoin(fs.fsPath, bucket, prefix, uploadsJSONFile)
 	_, err := fsStatFile(uploadsIDPath)
 	if err != nil {
-		if errorCause(err) == errFileNotFound {
+		if errors.Cause(err) == errFileNotFound {
 			return false
 		}
 		errorIf(err, "Unable to access uploads.json "+uploadsIDPath)
@@ -91,13 +85,13 @@ func (fs fsObjects) addUploadID(bucket, object, uploadID string, initiated time.
 
 	_, err := uploadIDs.ReadFrom(rwlk)
 	// For all unexpected errors, we return.
-	if err != nil && errorCause(err) != io.EOF {
+	if err != nil && errors.Cause(err) != io.EOF {
 		return err
 	}
 
 	// If we couldn't read anything, we assume a default
 	// (empty) upload info.
-	if errorCause(err) == io.EOF {
+	if errors.Cause(err) == io.EOF {
 		uploadIDs = newUploadsV1("fs")
 	}
 
@@ -110,14 +104,14 @@ func (fs fsObjects) addUploadID(bucket, object, uploadID string, initiated time.
 }
 
 // listMultipartUploadIDs - list all the upload ids from a marker up to 'count'.
-func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarker string, count int) ([]uploadMetadata, bool, error) {
-	var uploads []uploadMetadata
+func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarker string, count int) ([]MultipartInfo, bool, error) {
+	var uploads []MultipartInfo
 
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
 	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucketName, objectName))
 	if err := objectMPartPathLock.GetRLock(globalListingTimeout); err != nil {
-		return nil, false, traceError(err)
+		return nil, false, errors.Trace(err)
 	}
 	defer objectMPartPathLock.RUnlock()
 
@@ -127,7 +121,7 @@ func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarke
 		if err == errFileNotFound || err == errFileAccessDenied {
 			return nil, true, nil
 		}
-		return nil, false, traceError(err)
+		return nil, false, errors.Trace(err)
 	}
 	defer fs.rwPool.Close(pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadsPath))
 
@@ -149,7 +143,7 @@ func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarke
 	}
 
 	for index < len(uploadIDs.Uploads) {
-		uploads = append(uploads, uploadMetadata{
+		uploads = append(uploads, MultipartInfo{
 			Object:    objectName,
 			UploadID:  uploadIDs.Uploads[index].UploadID,
 			Initiated: uploadIDs.Uploads[index].Initiated,
@@ -191,7 +185,7 @@ func (fs fsObjects) listMultipartUploadsCleanup(bucket, prefix, keyMarker, uploa
 		multipartMarkerPath = pathJoin(bucket, keyMarker)
 	}
 
-	var uploads []uploadMetadata
+	var uploads []MultipartInfo
 	var err error
 	var eof bool
 
@@ -235,7 +229,7 @@ func (fs fsObjects) listMultipartUploadsCleanup(bucket, prefix, keyMarker, uploa
 			// For any walk error return right away.
 			if walkResult.err != nil {
 				// File not found or Disk not found is a valid case.
-				if isErrIgnored(walkResult.err, fsTreeWalkIgnoredErrs...) {
+				if errors.IsErrIgnored(walkResult.err, fsTreeWalkIgnoredErrs...) {
 					eof = true
 					break
 				}
@@ -244,7 +238,7 @@ func (fs fsObjects) listMultipartUploadsCleanup(bucket, prefix, keyMarker, uploa
 
 			entry := strings.TrimPrefix(walkResult.entry, retainSlash(bucket))
 			if hasSuffix(walkResult.entry, slashSeparator) {
-				uploads = append(uploads, uploadMetadata{
+				uploads = append(uploads, MultipartInfo{
 					Object: entry,
 				})
 				maxUploads--
@@ -257,7 +251,7 @@ func (fs fsObjects) listMultipartUploadsCleanup(bucket, prefix, keyMarker, uploa
 				continue
 			}
 
-			var tmpUploads []uploadMetadata
+			var tmpUploads []MultipartInfo
 			var end bool
 			uploadIDMarker = ""
 
@@ -372,7 +366,7 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st
 	uploadsPath := pathJoin(bucket, object, uploadsJSONFile)
 	rwlk, err := fs.rwPool.Create(pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadsPath))
 	if err != nil {
-		return "", toObjectErr(traceError(err), bucket, object)
+		return "", toObjectErr(errors.Trace(err), bucket, object)
 	}
 	defer rwlk.Close()
 
@@ -380,7 +374,7 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st
 	fsMetaPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, fsMetaJSONFile)
 	metaFile, err := fs.rwPool.Create(fsMetaPath)
 	if err != nil {
-		return "", toObjectErr(traceError(err), bucket, object)
+		return "", toObjectErr(errors.Trace(err), bucket, object)
 	}
 	defer metaFile.Close()
 
@@ -442,7 +436,9 @@ func partToAppend(fsMeta fsMetaV1, fsAppendMeta fsMetaV1) (part objectPartInfo,
 // CopyObjectPart - similar to PutObjectPart but reads data from an existing
 // object. Internally incoming data is written to '.minio.sys/tmp' location
 // and safely renamed to '.minio.sys/multipart' for reach parts.
-func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64) (pi PartInfo, e error) {
+func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int,
+	startOffset int64, length int64, metadata map[string]string) (pi PartInfo, e error) {
+
 	if err := checkNewMultipartArgs(srcBucket, srcObject, fs); err != nil {
 		return pi, err
 	}
@@ -490,7 +486,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
 
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < 0 {
-		return pi, toObjectErr(traceError(errInvalidArgument))
+		return pi, toObjectErr(errors.Trace(errInvalidArgument))
 	}
 
 	// Hold the lock so that two parallel complete-multipart-uploads
@@ -505,9 +501,9 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
 	uploadsPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, object, uploadsJSONFile)
 	if _, err := fs.rwPool.Open(uploadsPath); err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return pi, traceError(InvalidUploadID{UploadID: uploadID})
+			return pi, errors.Trace(InvalidUploadID{UploadID: uploadID})
 		}
-		return pi, toObjectErr(traceError(err), bucket, object)
+		return pi, toObjectErr(errors.Trace(err), bucket, object)
 	}
 	defer fs.rwPool.Close(uploadsPath)
 
@@ -518,9 +514,9 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
 	rwlk, err := fs.rwPool.Write(fsMetaPath)
 	if err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return pi, traceError(InvalidUploadID{UploadID: uploadID})
+			return pi, errors.Trace(InvalidUploadID{UploadID: uploadID})
 		}
-		return pi, toObjectErr(traceError(err), bucket, object)
+		return pi, toObjectErr(errors.Trace(err), bucket, object)
 	}
 	defer rwlk.Close()
 
@@ -550,7 +546,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
 	// bytes than specified in request header.
 	if bytesWritten < data.Size() {
 		fsRemoveFile(fsPartPath)
-		return pi, traceError(IncompleteBody{})
+		return pi, errors.Trace(IncompleteBody{})
 	}
 
 	// Delete temporary part in case of failure. If
@@ -618,9 +614,9 @@ func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberM
 	if err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
 			// On windows oddly this is returned.
-			return lpi, traceError(InvalidUploadID{UploadID: uploadID})
+			return lpi, errors.Trace(InvalidUploadID{UploadID: uploadID})
 		}
-		return lpi, toObjectErr(traceError(err), bucket, object)
+		return lpi, toObjectErr(errors.Trace(err), bucket, object)
 	}
 	defer fs.rwPool.Close(fsMetaPath)
 
@@ -695,7 +691,7 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
 	// do not leave a stale uploads.json behind.
 	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
 	if err := objectMPartPathLock.GetRLock(globalListingTimeout); err != nil {
-		return lpi, traceError(err)
+		return lpi, errors.Trace(err)
 	}
 	defer objectMPartPathLock.RUnlock()
 
@@ -713,14 +709,14 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
 // md5sums of all the parts.
 //
 // Implements S3 compatible Complete multipart API.
-func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (oi ObjectInfo, e error) {
+func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (oi ObjectInfo, e error) {
 	if err := checkCompleteMultipartArgs(bucket, object, fs); err != nil {
 		return oi, err
 	}
 
 	// Check if an object is present as one of the parent dir.
 	if fs.parentDirIsObject(bucket, pathutil.Dir(object)) {
-		return oi, toObjectErr(traceError(errFileAccessDenied), bucket, object)
+		return oi, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object)
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
@@ -747,7 +743,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		if removeObjectDir {
 			basePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket)
 			derr := fsDeleteFile(basePath, pathJoin(basePath, object))
-			if derr = errorCause(derr); derr != nil {
+			if derr = errors.Cause(derr); derr != nil {
 				// In parallel execution, CompleteMultipartUpload could have deleted temporary
 				// state files/directory, it is safe to ignore errFileNotFound
 				if derr != errFileNotFound {
@@ -762,9 +758,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	rlk, err := fs.rwPool.Open(fsMetaPathMultipart)
 	if err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return oi, traceError(InvalidUploadID{UploadID: uploadID})
+			return oi, errors.Trace(InvalidUploadID{UploadID: uploadID})
 		}
-		return oi, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(errors.Trace(err), bucket, object)
 	}
 
 	// Disallow any parallel abort or complete multipart operations.
@@ -772,9 +768,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	if err != nil {
 		fs.rwPool.Close(fsMetaPathMultipart)
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return oi, traceError(InvalidUploadID{UploadID: uploadID})
+			return oi, errors.Trace(InvalidUploadID{UploadID: uploadID})
 		}
-		return oi, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(errors.Trace(err), bucket, object)
 	}
 	defer rwlk.Close()
 
@@ -792,18 +788,18 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
 		if partIdx == -1 {
 			fs.rwPool.Close(fsMetaPathMultipart)
-			return oi, traceError(InvalidPart{})
+			return oi, errors.Trace(InvalidPart{})
 		}
 
 		if fsMeta.Parts[partIdx].ETag != part.ETag {
 			fs.rwPool.Close(fsMetaPathMultipart)
-			return oi, traceError(InvalidPart{})
+			return oi, errors.Trace(InvalidPart{})
 		}
 
 		// All parts except the last part has to be atleast 5MB.
 		if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
 			fs.rwPool.Close(fsMetaPathMultipart)
-			return oi, traceError(PartTooSmall{
+			return oi, errors.Trace(PartTooSmall{
 				PartNumber: part.PartNumber,
 				PartSize:   fsMeta.Parts[partIdx].Size,
 				PartETag:   part.ETag,
@@ -821,7 +817,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		// renamed to the main name-space.
 		if (i < len(parts)-1) && partSize != fsMeta.Parts[partIdx].Size {
 			fs.rwPool.Close(fsMetaPathMultipart)
-			return oi, traceError(PartsSizeUnequal{})
+			return oi, errors.Trace(PartsSizeUnequal{})
 		}
 	}
 
@@ -831,7 +827,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	metaFile, err := fs.rwPool.Create(fsMetaPath)
 	if err != nil {
 		fs.rwPool.Close(fsMetaPathMultipart)
-		return oi, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(errors.Trace(err), bucket, object)
 	}
 	defer metaFile.Close()
 
@@ -877,9 +873,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 			if err != nil {
 				fs.rwPool.Close(fsMetaPathMultipart)
 				if err == errFileNotFound {
-					return oi, traceError(InvalidPart{})
+					return oi, errors.Trace(InvalidPart{})
 				}
-				return oi, toObjectErr(traceError(err), minioMetaMultipartBucket, partSuffix)
+				return oi, toObjectErr(errors.Trace(err), minioMetaMultipartBucket, partSuffix)
 			}
 
 			// No need to hold a lock, this is a unique file and will be only written
@@ -889,7 +885,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 			if err != nil {
 				reader.Close()
 				fs.rwPool.Close(fsMetaPathMultipart)
-				return oi, toObjectErr(traceError(err), bucket, object)
+				return oi, toObjectErr(errors.Trace(err), bucket, object)
 			}
 
 			_, err = io.CopyBuffer(wfile, reader, buf)
@@ -897,7 +893,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 				wfile.Close()
 				reader.Close()
 				fs.rwPool.Close(fsMetaPathMultipart)
-				return oi, toObjectErr(traceError(err), bucket, object)
+				return oi, toObjectErr(errors.Trace(err), bucket, object)
 			}
 
 			wfile.Close()
@@ -988,7 +984,7 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 		if removeObjectDir {
 			basePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket)
 			derr := fsDeleteFile(basePath, pathJoin(basePath, object))
-			if derr = errorCause(derr); derr != nil {
+			if derr = errors.Cause(derr); derr != nil {
 				// In parallel execution, AbortMultipartUpload could have deleted temporary
 				// state files/directory, it is safe to ignore errFileNotFound
 				if derr != errFileNotFound {
@@ -1002,9 +998,9 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 	fsMetaPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, fsMetaJSONFile)
 	if _, err := fs.rwPool.Open(fsMetaPath); err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return traceError(InvalidUploadID{UploadID: uploadID})
+			return errors.Trace(InvalidUploadID{UploadID: uploadID})
 		}
-		return toObjectErr(traceError(err), bucket, object)
+		return toObjectErr(errors.Trace(err), bucket, object)
 	}
 
 	uploadsPath := pathJoin(bucket, object, uploadsJSONFile)
@@ -1012,9 +1008,9 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 	if err != nil {
 		fs.rwPool.Close(fsMetaPath)
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return traceError(InvalidUploadID{UploadID: uploadID})
+			return errors.Trace(InvalidUploadID{UploadID: uploadID})
 		}
-		return toObjectErr(traceError(err), bucket, object)
+		return toObjectErr(errors.Trace(err), bucket, object)
 	}
 	defer rwlk.Close()
 
@@ -1041,61 +1037,3 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 
 	return nil
 }
-
-// Removes multipart uploads if any older than `expiry` duration in a given bucket.
-func (fs fsObjects) cleanupStaleMultipartUpload(bucket string, expiry time.Duration) (err error) {
-	var lmi ListMultipartsInfo
-	var st os.FileInfo
-	for {
-		// List multipart uploads in a bucket 1000 at a time
-		prefix := ""
-		lmi, err = fs.listMultipartUploadsCleanup(bucket, prefix, lmi.KeyMarker, lmi.UploadIDMarker, "", 1000)
-		if err != nil {
-			errorIf(err, "Unable to list uploads")
-			return err
-		}
-
-		// Remove uploads (and its parts) older than expiry duration.
-		for _, upload := range lmi.Uploads {
-			uploadIDPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, upload.Object, upload.UploadID)
-			if st, err = fsStatDir(uploadIDPath); err != nil {
-				errorIf(err, "Failed to lookup uploads directory path %s", uploadIDPath)
-				continue
-			}
-			if time.Since(st.ModTime()) > expiry {
-				fs.AbortMultipartUpload(bucket, upload.Object, upload.UploadID)
-			}
-		}
-
-		// No more incomplete uploads remain, break and return.
-		if !lmi.IsTruncated {
-			break
-		}
-	}
-
-	return nil
-}
-
-// Removes multipart uploads if any older than `expiry` duration
-// on all buckets for every `cleanupInterval`, this function is
-// blocking and should be run in a go-routine.
-func (fs fsObjects) cleanupStaleMultipartUploads(cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
-	ticker := time.NewTicker(cleanupInterval)
-	for {
-		select {
-		case <-doneCh:
-			// Stop the timer.
-			ticker.Stop()
-			return
-		case <-ticker.C:
-			bucketInfos, err := fs.ListBuckets()
-			if err != nil {
-				errorIf(err, "Unable to list buckets")
-				continue
-			}
-			for _, bucketInfo := range bucketInfos {
-				fs.cleanupStaleMultipartUpload(bucketInfo.Name, expiry)
-			}
-		}
-	}
-}

cmd/fs-v1-multipart_test.go

@@ -22,8 +22,11 @@ import (
 	"path/filepath"
 	"testing"
 	"time"
+
+	"github.com/minio/minio/pkg/errors"
 )
 
+// Tests cleanup multipart uploads for filesystem backend.
 func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
@@ -45,7 +48,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 		t.Fatal("Unexpected err: ", err)
 	}
 
-	go fs.cleanupStaleMultipartUploads(20*time.Millisecond, 0, globalServiceDoneCh)
+	go cleanupStaleMultipartUploads(20*time.Millisecond, 0, obj, fs.listMultipartUploadsCleanup, globalServiceDoneCh)
 
 	// Wait for 40ms such that - we have given enough time for
 	// cleanup routine to kick in.
@@ -56,7 +59,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 
 	// Check if upload id was already purged.
 	if err = obj.AbortMultipartUpload(bucketName, objectName, uploadID); err != nil {
-		err = errorCause(err)
+		err = errors.Cause(err)
 		if _, ok := err.(InvalidUploadID); !ok {
 			t.Fatal("Unexpected err: ", err)
 		}
@@ -87,13 +90,13 @@ func TestFSCleanupMultipartUpload(t *testing.T) {
 		t.Fatal("Unexpected err: ", err)
 	}
 
-	if err = fs.cleanupStaleMultipartUpload(bucketName, 0); err != nil {
+	if err = cleanupStaleMultipartUpload(bucketName, 0, obj, fs.listMultipartUploadsCleanup); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
 	// Check if upload id was already purged.
 	if err = obj.AbortMultipartUpload(bucketName, objectName, uploadID); err != nil {
-		err = errorCause(err)
+		err = errors.Cause(err)
 		if _, ok := err.(InvalidUploadID); !ok {
 			t.Fatal("Unexpected err: ", err)
 		}
@@ -122,7 +125,7 @@ func TestFSWriteUploadJSON(t *testing.T) {
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	_, err = obj.NewMultipartUpload(bucketName, objectName, nil)
 	if err != nil {
-		if _, ok := errorCause(err).(BucketNotFound); !ok {
+		if _, ok := errors.Cause(err).(BucketNotFound); !ok {
 			t.Fatal("Unexpected err: ", err)
 		}
 	}
@@ -146,7 +149,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 	// Test with disk removed.
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	if _, err := fs.NewMultipartUpload(bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}); err != nil {
-		if !isSameType(errorCause(err), BucketNotFound{}) {
+		if !isSameType(errors.Cause(err), BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
 	}
@@ -184,7 +187,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	_, err = fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum))
-	if !isSameType(errorCause(err), BucketNotFound{}) {
+	if !isSameType(errors.Cause(err), BucketNotFound{}) {
 		t.Fatal("Unexpected error ", err)
 	}
 }
@@ -216,11 +219,11 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 		t.Fatal("Unexpected error ", err)
 	}
 
-	parts := []completePart{{PartNumber: 1, ETag: md5Hex}}
+	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
 
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	if _, err := fs.CompleteMultipartUpload(bucketName, objectName, uploadID, parts); err != nil {
-		if !isSameType(errorCause(err), BucketNotFound{}) {
+		if !isSameType(errors.Cause(err), BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
 	}
@@ -253,7 +256,7 @@ func TestCompleteMultipartUpload(t *testing.T) {
 		t.Fatal("Unexpected error ", err)
 	}
 
-	parts := []completePart{{PartNumber: 1, ETag: md5Hex}}
+	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
 
 	if _, err := fs.CompleteMultipartUpload(bucketName, objectName, uploadID, parts); err != nil {
 		t.Fatal("Unexpected error ", err)
@@ -323,7 +326,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	if _, err := fs.ListMultipartUploads(bucketName, objectName, "", "", "", 1000); err != nil {
-		if !isSameType(errorCause(err), BucketNotFound{}) {
+		if !isSameType(errors.Cause(err), BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
 	}

cmd/fs-v1.go

@@ -27,6 +27,7 @@ import (
 	"sort"
 	"syscall"
 
+	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 )
@@ -155,8 +156,8 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		return nil, fmt.Errorf("Unable to initialize event notification. %s", err)
 	}
 
-	// Start background process to cleanup old files in `.minio.sys`.
-	go fs.cleanupStaleMultipartUploads(fsMultipartCleanupInterval, fsMultipartExpiry, globalServiceDoneCh)
+	// Start background process to cleanup old multipart objects in `.minio.sys`.
+	go cleanupStaleMultipartUploads(multipartCleanupInterval, multipartExpiry, fs, fs.listMultipartUploadsCleanup, globalServiceDoneCh)
 
 	// Return successfully initialized object layer.
 	return fs, nil
@@ -190,7 +191,7 @@ func (fs fsObjects) StorageInfo() StorageInfo {
 func (fs fsObjects) getBucketDir(bucket string) (string, error) {
 	// Verify if bucket is valid.
 	if !IsValidBucketName(bucket) {
-		return "", traceError(BucketNameInvalid{Bucket: bucket})
+		return "", errors.Trace(BucketNameInvalid{Bucket: bucket})
 	}
 
 	bucketDir := pathJoin(fs.fsPath, bucket)
@@ -242,12 +243,12 @@ func (fs fsObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
 // ListBuckets - list all s3 compatible buckets (directories) at fsPath.
 func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
 	if err := checkPathLength(fs.fsPath); err != nil {
-		return nil, traceError(err)
+		return nil, errors.Trace(err)
 	}
 	var bucketInfos []BucketInfo
 	entries, err := readDir((fs.fsPath))
 	if err != nil {
-		return nil, toObjectErr(traceError(errDiskNotFound))
+		return nil, toObjectErr(errors.Trace(errDiskNotFound))
 	}
 
 	for _, entry := range entries {
@@ -330,7 +331,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 		var wlk *lock.LockedFile
 		wlk, err = fs.rwPool.Write(fsMetaPath)
 		if err != nil {
-			return oi, toObjectErr(traceError(err), srcBucket, srcObject)
+			return oi, toObjectErr(errors.Trace(err), srcBucket, srcObject)
 		}
 		// This close will allow for locks to be synchronized on `fs.json`.
 		defer wlk.Close()
@@ -395,25 +396,25 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
 
 	// Offset cannot be negative.
 	if offset < 0 {
-		return toObjectErr(traceError(errUnexpected), bucket, object)
+		return toObjectErr(errors.Trace(errUnexpected), bucket, object)
 	}
 
 	// Writer cannot be nil.
 	if writer == nil {
-		return toObjectErr(traceError(errUnexpected), bucket, object)
+		return toObjectErr(errors.Trace(errUnexpected), bucket, object)
 	}
 
 	// If its a directory request, we return an empty body.
 	if hasSuffix(object, slashSeparator) {
 		_, err = writer.Write([]byte(""))
-		return toObjectErr(traceError(err), bucket, object)
+		return toObjectErr(errors.Trace(err), bucket, object)
 	}
 
 	if bucket != minioMetaBucket {
 		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
 		_, err = fs.rwPool.Open(fsMetaPath)
 		if err != nil && err != errFileNotFound {
-			return toObjectErr(traceError(err), bucket, object)
+			return toObjectErr(errors.Trace(err), bucket, object)
 		}
 		defer fs.rwPool.Close(fsMetaPath)
 	}
@@ -438,7 +439,7 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
 
 	// Reply back invalid range if the input offset and length fall out of range.
 	if offset > size || offset+length > size {
-		return traceError(InvalidRange{offset, length, size})
+		return errors.Trace(InvalidRange{offset, length, size})
 	}
 
 	// Allocate a staging buffer.
@@ -446,19 +447,23 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
 
 	_, err = io.CopyBuffer(writer, io.LimitReader(reader, length), buf)
 
-	return toObjectErr(traceError(err), bucket, object)
+	return toObjectErr(errors.Trace(err), bucket, object)
 }
 
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
 func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
 	fsMeta := fsMetaV1{}
-	if hasSuffix(object, slashSeparator) {
-		// Directory call needs to arrive with object ending with "/".
-		fi, err := fsStatDir(pathJoin(fs.fsPath, bucket, object))
-		if err != nil {
-			return oi, toObjectErr(err, bucket, object)
+	fi, err := fsStatDir(pathJoin(fs.fsPath, bucket, object))
+	if err != nil && errors.Cause(err) != errFileAccessDenied {
+		return oi, toObjectErr(err, bucket, object)
+	}
+	if fi != nil {
+		// If file found and request was with object ending with "/", consider it
+		// as directory and return object info
+		if hasSuffix(object, slashSeparator) {
+			return fsMeta.ToObjectInfo(bucket, object, fi), nil
 		}
-		return fsMeta.ToObjectInfo(bucket, object, fi), nil
+		return oi, toObjectErr(errFileNotFound, bucket, object)
 	}
 
 	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
@@ -473,7 +478,7 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error
 			// `fs.json` can be empty due to previously failed
 			// PutObject() transaction, if we arrive at such
 			// a situation we just ignore and continue.
-			if errorCause(rerr) != io.EOF {
+			if errors.Cause(rerr) != io.EOF {
 				return oi, toObjectErr(rerr, bucket, object)
 			}
 		}
@@ -481,11 +486,11 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error
 
 	// Ignore if `fs.json` is not available, this is true for pre-existing data.
 	if err != nil && err != errFileNotFound {
-		return oi, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(errors.Trace(err), bucket, object)
 	}
 
 	// Stat the file to get file size.
-	fi, err := fsStatFile(pathJoin(fs.fsPath, bucket, object))
+	fi, err = fsStatFile(pathJoin(fs.fsPath, bucket, object))
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object)
 	}
@@ -497,14 +502,14 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error
 func checkBucketAndObjectNamesFS(bucket, object string) error {
 	// Verify if bucket is valid.
 	if !IsValidBucketName(bucket) {
-		return traceError(BucketNameInvalid{Bucket: bucket})
+		return errors.Trace(BucketNameInvalid{Bucket: bucket})
 	}
 	// Verify if object is valid.
 	if len(object) == 0 {
-		return traceError(ObjectNameInvalid{Bucket: bucket, Object: object})
+		return errors.Trace(ObjectNameInvalid{Bucket: bucket, Object: object})
 	}
 	if !IsValidObjectPrefix(object) {
-		return traceError(ObjectNameInvalid{Bucket: bucket, Object: object})
+		return errors.Trace(ObjectNameInvalid{Bucket: bucket, Object: object})
 	}
 	return nil
 }
@@ -568,7 +573,7 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
 	if isObjectDir(object, data.Size()) {
 		// Check if an object is present as one of the parent dir.
 		if fs.parentDirIsObject(bucket, path.Dir(object)) {
-			return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
+			return ObjectInfo{}, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object)
 		}
 		if err = fsMkdirAll(pathJoin(fs.fsPath, bucket, object)); err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
@@ -586,12 +591,12 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
 
 	// Check if an object is present as one of the parent dir.
 	if fs.parentDirIsObject(bucket, path.Dir(object)) {
-		return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
+		return ObjectInfo{}, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object)
 	}
 
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < 0 {
-		return ObjectInfo{}, traceError(errInvalidArgument)
+		return ObjectInfo{}, errors.Trace(errInvalidArgument)
 	}
 
 	var wlk *lock.LockedFile
@@ -600,7 +605,7 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
 		fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fsMetaJSONFile)
 		wlk, err = fs.rwPool.Create(fsMetaPath)
 		if err != nil {
-			return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+			return ObjectInfo{}, toObjectErr(errors.Trace(err), bucket, object)
 		}
 		// This close will allow for locks to be synchronized on `fs.json`.
 		defer wlk.Close()
@@ -639,7 +644,7 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
 	// bytes than specified in request header.
 	if bytesWritten < data.Size() {
 		fsRemoveFile(fsTmpObjPath)
-		return ObjectInfo{}, traceError(IncompleteBody{})
+		return ObjectInfo{}, errors.Trace(IncompleteBody{})
 	}
 
 	// Delete the temporary object in the case of a
@@ -690,7 +695,7 @@ func (fs fsObjects) DeleteObject(bucket, object string) error {
 			defer rwlk.Close()
 		}
 		if lerr != nil && lerr != errFileNotFound {
-			return toObjectErr(traceError(lerr), bucket, object)
+			return toObjectErr(errors.Trace(lerr), bucket, object)
 		}
 	}
 
@@ -702,7 +707,7 @@ func (fs fsObjects) DeleteObject(bucket, object string) error {
 	if bucket != minioMetaBucket {
 		// Delete the metadata object.
 		err := fsDeleteFile(minioMetaBucketDir, fsMetaPath)
-		if err != nil && errorCause(err) != errFileNotFound {
+		if err != nil && errors.Cause(err) != errFileNotFound {
 			return toObjectErr(err, bucket, object)
 		}
 	}
@@ -743,7 +748,7 @@ func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
 	rlk, err := fs.rwPool.Open(fsMetaPath)
 	// Ignore if `fs.json` is not available, this is true for pre-existing data.
 	if err != nil && err != errFileNotFound {
-		return "", toObjectErr(traceError(err), bucket, entry)
+		return "", toObjectErr(errors.Trace(err), bucket, entry)
 	}
 
 	// If file is not found, we don't need to proceed forward.
@@ -757,7 +762,7 @@ func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
 	// Fetch the size of the underlying file.
 	fi, err := rlk.LockedFile.Stat()
 	if err != nil {
-		return "", toObjectErr(traceError(err), bucket, entry)
+		return "", toObjectErr(errors.Trace(err), bucket, entry)
 	}
 
 	// `fs.json` can be empty due to previously failed
@@ -771,12 +776,12 @@ func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
 	// make sure the underlying offsets don't move.
 	fsMetaBuf, err := ioutil.ReadAll(io.NewSectionReader(rlk.LockedFile, 0, fi.Size()))
 	if err != nil {
-		return "", traceError(err)
+		return "", errors.Trace(err)
 	}
 
 	// Check if FS metadata is valid, if not return error.
 	if !isFSMetaValid(parseFSVersion(fsMetaBuf), parseFSFormat(fsMetaBuf)) {
-		return "", toObjectErr(traceError(errCorruptedFormat), bucket, entry)
+		return "", toObjectErr(errors.Trace(errCorruptedFormat), bucket, entry)
 	}
 
 	return extractETag(parseFSMetaMap(fsMetaBuf)), nil
@@ -898,7 +903,7 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 		// For any walk error return right away.
 		if walkResult.err != nil {
 			// File not found is a valid case.
-			if errorCause(walkResult.err) == errFileNotFound {
+			if errors.Cause(walkResult.err) == errFileNotFound {
 				return loi, nil
 			}
 			return loi, toObjectErr(walkResult.err, bucket, prefix)
@@ -939,25 +944,25 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 
 // HealObject - no-op for fs. Valid only for XL.
 func (fs fsObjects) HealObject(bucket, object string) (int, int, error) {
-	return 0, 0, traceError(NotImplemented{})
+	return 0, 0, errors.Trace(NotImplemented{})
 }
 
 // HealBucket - no-op for fs, Valid only for XL.
 func (fs fsObjects) HealBucket(bucket string) error {
-	return traceError(NotImplemented{})
+	return errors.Trace(NotImplemented{})
 }
 
 // ListObjectsHeal - list all objects to be healed. Valid only for XL
 func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	return loi, traceError(NotImplemented{})
+	return loi, errors.Trace(NotImplemented{})
 }
 
 // ListBucketsHeal - list all buckets to be healed. Valid only for XL
 func (fs fsObjects) ListBucketsHeal() ([]BucketInfo, error) {
-	return []BucketInfo{}, traceError(NotImplemented{})
+	return []BucketInfo{}, errors.Trace(NotImplemented{})
 }
 
 func (fs fsObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
 	delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
-	return lmi, traceError(NotImplemented{})
+	return lmi, errors.Trace(NotImplemented{})
 }

cmd/fs-v1_test.go

@@ -22,6 +22,8 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/minio/minio/pkg/errors"
 )
 
 // Tests for if parent directory is object
@@ -178,7 +180,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 
 	// Test with inexistant bucket
 	_, err = fs.GetBucketInfo("a")
-	if !isSameType(errorCause(err), BucketNameInvalid{}) {
+	if !isSameType(errors.Cause(err), BucketNameInvalid{}) {
 		t.Fatal("BucketNameInvalid error not returned")
 	}
 
@@ -186,7 +188,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 
 	_, err = fs.GetBucketInfo(bucketName)
-	if !isSameType(errorCause(err), BucketNotFound{}) {
+	if !isSameType(errors.Cause(err), BucketNotFound{}) {
 		t.Fatal("BucketNotFound error not returned")
 	}
 }
@@ -209,7 +211,7 @@ func TestFSPutObject(t *testing.T) {
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
-	if _, ok := errorCause(err).(BucketNotFound); !ok {
+	if _, ok := errors.Cause(err).(BucketNotFound); !ok {
 		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
 	}
 
@@ -218,7 +220,7 @@ func TestFSPutObject(t *testing.T) {
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
-	if _, ok := errorCause(err).(BucketNotFound); !ok {
+	if _, ok := errors.Cause(err).(BucketNotFound); !ok {
 		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
 	}
 
@@ -230,7 +232,7 @@ func TestFSPutObject(t *testing.T) {
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backend corruption occurred")
 	}
-	if nerr, ok := errorCause(err).(PrefixAccessDenied); !ok {
+	if nerr, ok := errors.Cause(err).(PrefixAccessDenied); !ok {
 		t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
 	} else {
 		if nerr.Bucket != "bucket" {
@@ -245,7 +247,7 @@ func TestFSPutObject(t *testing.T) {
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backned corruption occurred")
 	}
-	if nerr, ok := errorCause(err).(PrefixAccessDenied); !ok {
+	if nerr, ok := errors.Cause(err).(PrefixAccessDenied); !ok {
 		t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
 	} else {
 		if nerr.Bucket != "bucket" {
@@ -272,19 +274,19 @@ func TestFSDeleteObject(t *testing.T) {
 	obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
 
 	// Test with invalid bucket name
-	if err := fs.DeleteObject("fo", objectName); !isSameType(errorCause(err), BucketNameInvalid{}) {
+	if err := fs.DeleteObject("fo", objectName); !isSameType(errors.Cause(err), BucketNameInvalid{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with bucket does not exist
-	if err := fs.DeleteObject("foobucket", "fooobject"); !isSameType(errorCause(err), BucketNotFound{}) {
+	if err := fs.DeleteObject("foobucket", "fooobject"); !isSameType(errors.Cause(err), BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with invalid object name
-	if err := fs.DeleteObject(bucketName, "\\"); !isSameType(errorCause(err), ObjectNameInvalid{}) {
+	if err := fs.DeleteObject(bucketName, "\\"); !isSameType(errors.Cause(err), ObjectNameInvalid{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with object does not exist.
-	if err := fs.DeleteObject(bucketName, "foooobject"); !isSameType(errorCause(err), ObjectNotFound{}) {
+	if err := fs.DeleteObject(bucketName, "foooobject"); !isSameType(errors.Cause(err), ObjectNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with valid condition
@@ -295,7 +297,7 @@ func TestFSDeleteObject(t *testing.T) {
 	// Delete object should err disk not found.
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	if err := fs.DeleteObject(bucketName, objectName); err != nil {
-		if !isSameType(errorCause(err), BucketNotFound{}) {
+		if !isSameType(errors.Cause(err), BucketNotFound{}) {
 			t.Fatal("Unexpected error: ", err)
 		}
 	}
@@ -318,11 +320,11 @@ func TestFSDeleteBucket(t *testing.T) {
 	}
 
 	// Test with an invalid bucket name
-	if err = fs.DeleteBucket("fo"); !isSameType(errorCause(err), BucketNameInvalid{}) {
+	if err = fs.DeleteBucket("fo"); !isSameType(errors.Cause(err), BucketNameInvalid{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with an inexistant bucket
-	if err = fs.DeleteBucket("foobucket"); !isSameType(errorCause(err), BucketNotFound{}) {
+	if err = fs.DeleteBucket("foobucket"); !isSameType(errors.Cause(err), BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with a valid case
@@ -335,7 +337,7 @@ func TestFSDeleteBucket(t *testing.T) {
 	// Delete bucket should get error disk not found.
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	if err = fs.DeleteBucket(bucketName); err != nil {
-		if !isSameType(errorCause(err), BucketNotFound{}) {
+		if !isSameType(errors.Cause(err), BucketNotFound{}) {
 			t.Fatal("Unexpected error: ", err)
 		}
 	}
@@ -378,7 +380,7 @@ func TestFSListBuckets(t *testing.T) {
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 
 	if _, err := fs.ListBuckets(); err != nil {
-		if errorCause(err) != errDiskNotFound {
+		if errors.Cause(err) != errDiskNotFound {
 			t.Fatal("Unexpected error: ", err)
 		}
 	}
@@ -386,7 +388,7 @@ func TestFSListBuckets(t *testing.T) {
 	longPath := fmt.Sprintf("%0256d", 1)
 	fs.fsPath = longPath
 	if _, err := fs.ListBuckets(); err != nil {
-		if errorCause(err) != errFileNameTooLong {
+		if errors.Cause(err) != errFileNameTooLong {
 			t.Fatal("Unexpected error: ", err)
 		}
 	}
@@ -399,7 +401,7 @@ func TestFSHealObject(t *testing.T) {
 
 	obj := initFSObjects(disk, t)
 	_, _, err := obj.HealObject("bucket", "object")
-	if err == nil || !isSameType(errorCause(err), NotImplemented{}) {
+	if err == nil || !isSameType(errors.Cause(err), NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}
 }
@@ -411,7 +413,7 @@ func TestFSListObjectsHeal(t *testing.T) {
 
 	obj := initFSObjects(disk, t)
 	_, err := obj.ListObjectsHeal("bucket", "prefix", "marker", "delimiter", 1000)
-	if err == nil || !isSameType(errorCause(err), NotImplemented{}) {
+	if err == nil || !isSameType(errors.Cause(err), NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}
 }

cmd/gateway-anonymous.go

@@ -1,74 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/tls"
-	"net"
-	"net/http"
-	"time"
-)
-
-func anonErrToObjectErr(statusCode int, params ...string) error {
-	bucket := ""
-	object := ""
-	if len(params) >= 1 {
-		bucket = params[0]
-	}
-	if len(params) == 2 {
-		object = params[1]
-	}
-
-	switch statusCode {
-	case http.StatusNotFound:
-		if object != "" {
-			return ObjectNotFound{bucket, object}
-		}
-		return BucketNotFound{Bucket: bucket}
-	case http.StatusBadRequest:
-		if object != "" {
-			return ObjectNameInvalid{bucket, object}
-		}
-		return BucketNameInvalid{Bucket: bucket}
-	case http.StatusForbidden:
-		fallthrough
-	case http.StatusUnauthorized:
-		return AllAccessDisabled{bucket, object}
-	}
-
-	return errUnexpected
-}
-
-// newCustomHTTPTransport returns a new http configuration
-// used while communicating with the cloud backends.
-// This sets the value for MaxIdleConns from 2 (go default) to
-// 100.
-func newCustomHTTPTransport() http.RoundTripper {
-	return &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-		}).DialContext,
-		MaxIdleConns:          100,
-		IdleConnTimeout:       90 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: globalRootCAs},
-		DisableCompression:    true,
-	}
-}

cmd/gateway-common.go

@@ -0,0 +1,325 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/http"
+
+	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/pkg/hash"
+
+	minio "github.com/minio/minio-go"
+)
+
+var (
+	// CanonicalizeETag provides canonicalizeETag function alias.
+	CanonicalizeETag = canonicalizeETag
+
+	// MustGetUUID function alias.
+	MustGetUUID = mustGetUUID
+
+	// ErrorIf provides errorIf function alias.
+	ErrorIf = errorIf
+
+	// FatalIf provides fatalIf function alias.
+	FatalIf = fatalIf
+)
+
+// AnonErrToObjectErr - converts standard http codes into meaningful object layer errors.
+func AnonErrToObjectErr(statusCode int, params ...string) error {
+	bucket := ""
+	object := ""
+	if len(params) >= 1 {
+		bucket = params[0]
+	}
+	if len(params) == 2 {
+		object = params[1]
+	}
+
+	switch statusCode {
+	case http.StatusNotFound:
+		if object != "" {
+			return ObjectNotFound{bucket, object}
+		}
+		return BucketNotFound{Bucket: bucket}
+	case http.StatusBadRequest:
+		if object != "" {
+			return ObjectNameInvalid{bucket, object}
+		}
+		return BucketNameInvalid{Bucket: bucket}
+	case http.StatusForbidden:
+		fallthrough
+	case http.StatusUnauthorized:
+		return AllAccessDisabled{bucket, object}
+	}
+
+	return errUnexpected
+}
+
+// FromMinioClientMetadata converts minio metadata to map[string]string
+func FromMinioClientMetadata(metadata map[string][]string) map[string]string {
+	mm := map[string]string{}
+	for k, v := range metadata {
+		mm[http.CanonicalHeaderKey(k)] = v[0]
+	}
+	return mm
+}
+
+// FromMinioClientObjectPart converts minio ObjectPart to PartInfo
+func FromMinioClientObjectPart(op minio.ObjectPart) PartInfo {
+	return PartInfo{
+		Size:         op.Size,
+		ETag:         canonicalizeETag(op.ETag),
+		LastModified: op.LastModified,
+		PartNumber:   op.PartNumber,
+	}
+}
+
+// FromMinioClientListPartsInfo converts minio ListObjectPartsResult to ListPartsInfo
+func FromMinioClientListPartsInfo(lopr minio.ListObjectPartsResult) ListPartsInfo {
+	// Convert minio ObjectPart to PartInfo
+	fromMinioClientObjectParts := func(parts []minio.ObjectPart) []PartInfo {
+		toParts := make([]PartInfo, len(parts))
+		for i, part := range parts {
+			toParts[i] = FromMinioClientObjectPart(part)
+		}
+		return toParts
+	}
+
+	return ListPartsInfo{
+		UploadID:             lopr.UploadID,
+		Bucket:               lopr.Bucket,
+		Object:               lopr.Key,
+		StorageClass:         "",
+		PartNumberMarker:     lopr.PartNumberMarker,
+		NextPartNumberMarker: lopr.NextPartNumberMarker,
+		MaxParts:             lopr.MaxParts,
+		IsTruncated:          lopr.IsTruncated,
+		EncodingType:         lopr.EncodingType,
+		Parts:                fromMinioClientObjectParts(lopr.ObjectParts),
+	}
+}
+
+// FromMinioClientListMultipartsInfo converts minio ListMultipartUploadsResult to ListMultipartsInfo
+func FromMinioClientListMultipartsInfo(lmur minio.ListMultipartUploadsResult) ListMultipartsInfo {
+	uploads := make([]MultipartInfo, len(lmur.Uploads))
+
+	for i, um := range lmur.Uploads {
+		uploads[i] = MultipartInfo{
+			Object:    um.Key,
+			UploadID:  um.UploadID,
+			Initiated: um.Initiated,
+		}
+	}
+
+	commonPrefixes := make([]string, len(lmur.CommonPrefixes))
+	for i, cp := range lmur.CommonPrefixes {
+		commonPrefixes[i] = cp.Prefix
+	}
+
+	return ListMultipartsInfo{
+		KeyMarker:          lmur.KeyMarker,
+		UploadIDMarker:     lmur.UploadIDMarker,
+		NextKeyMarker:      lmur.NextKeyMarker,
+		NextUploadIDMarker: lmur.NextUploadIDMarker,
+		MaxUploads:         int(lmur.MaxUploads),
+		IsTruncated:        lmur.IsTruncated,
+		Uploads:            uploads,
+		Prefix:             lmur.Prefix,
+		Delimiter:          lmur.Delimiter,
+		CommonPrefixes:     commonPrefixes,
+		EncodingType:       lmur.EncodingType,
+	}
+
+}
+
+// FromMinioClientObjectInfo converts minio ObjectInfo to gateway ObjectInfo
+func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
+	userDefined := FromMinioClientMetadata(oi.Metadata)
+	userDefined["Content-Type"] = oi.ContentType
+
+	return ObjectInfo{
+		Bucket:          bucket,
+		Name:            oi.Key,
+		ModTime:         oi.LastModified,
+		Size:            oi.Size,
+		ETag:            canonicalizeETag(oi.ETag),
+		UserDefined:     userDefined,
+		ContentType:     oi.ContentType,
+		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
+	}
+}
+
+// FromMinioClientListBucketV2Result converts minio ListBucketResult to ListObjectsInfo
+func FromMinioClientListBucketV2Result(bucket string, result minio.ListBucketV2Result) ListObjectsV2Info {
+	objects := make([]ObjectInfo, len(result.Contents))
+
+	for i, oi := range result.Contents {
+		objects[i] = FromMinioClientObjectInfo(bucket, oi)
+	}
+
+	prefixes := make([]string, len(result.CommonPrefixes))
+	for i, p := range result.CommonPrefixes {
+		prefixes[i] = p.Prefix
+	}
+
+	return ListObjectsV2Info{
+		IsTruncated: result.IsTruncated,
+		Prefixes:    prefixes,
+		Objects:     objects,
+
+		ContinuationToken:     result.ContinuationToken,
+		NextContinuationToken: result.NextContinuationToken,
+	}
+}
+
+// FromMinioClientListBucketResult converts minio ListBucketResult to ListObjectsInfo
+func FromMinioClientListBucketResult(bucket string, result minio.ListBucketResult) ListObjectsInfo {
+	objects := make([]ObjectInfo, len(result.Contents))
+
+	for i, oi := range result.Contents {
+		objects[i] = FromMinioClientObjectInfo(bucket, oi)
+	}
+
+	prefixes := make([]string, len(result.CommonPrefixes))
+	for i, p := range result.CommonPrefixes {
+		prefixes[i] = p.Prefix
+	}
+
+	return ListObjectsInfo{
+		IsTruncated: result.IsTruncated,
+		NextMarker:  result.NextMarker,
+		Prefixes:    prefixes,
+		Objects:     objects,
+	}
+}
+
+// FromMinioClientListBucketResultToV2Info converts minio ListBucketResult to ListObjectsV2Info
+func FromMinioClientListBucketResultToV2Info(bucket string, result minio.ListBucketResult) ListObjectsV2Info {
+	objects := make([]ObjectInfo, len(result.Contents))
+
+	for i, oi := range result.Contents {
+		objects[i] = FromMinioClientObjectInfo(bucket, oi)
+	}
+
+	prefixes := make([]string, len(result.CommonPrefixes))
+	for i, p := range result.CommonPrefixes {
+		prefixes[i] = p.Prefix
+	}
+
+	return ListObjectsV2Info{
+		IsTruncated:           result.IsTruncated,
+		Prefixes:              prefixes,
+		Objects:               objects,
+		ContinuationToken:     result.Marker,
+		NextContinuationToken: result.NextMarker,
+	}
+}
+
+// ToMinioClientMetadata converts metadata to map[string][]string
+func ToMinioClientMetadata(metadata map[string]string) map[string]string {
+	mm := make(map[string]string)
+	for k, v := range metadata {
+		mm[http.CanonicalHeaderKey(k)] = v
+	}
+	return mm
+}
+
+// ToMinioClientCompletePart converts CompletePart to minio CompletePart
+func ToMinioClientCompletePart(part CompletePart) minio.CompletePart {
+	return minio.CompletePart{
+		ETag:       part.ETag,
+		PartNumber: part.PartNumber,
+	}
+}
+
+// ToMinioClientCompleteParts converts []CompletePart to minio []CompletePart
+func ToMinioClientCompleteParts(parts []CompletePart) []minio.CompletePart {
+	mparts := make([]minio.CompletePart, len(parts))
+	for i, part := range parts {
+		mparts[i] = ToMinioClientCompletePart(part)
+	}
+	return mparts
+}
+
+// ErrorRespToObjectError converts Minio errors to minio object layer errors.
+func ErrorRespToObjectError(err error, params ...string) error {
+	if err == nil {
+		return nil
+	}
+
+	e, ok := err.(*errors.Error)
+	if !ok {
+		// Code should be fixed if this function is called without doing traceError()
+		// Else handling different situations in this function makes this function complicated.
+		errorIf(err, "Expected type *Error")
+		return err
+	}
+
+	err = e.Cause
+
+	bucket := ""
+	object := ""
+	if len(params) >= 1 {
+		bucket = params[0]
+	}
+	if len(params) == 2 {
+		object = params[1]
+	}
+
+	minioErr, ok := err.(minio.ErrorResponse)
+	if !ok {
+		// We don't interpret non Minio errors. As minio errors will
+		// have StatusCode to help to convert to object errors.
+		return e
+	}
+
+	switch minioErr.Code {
+	case "BucketAlreadyOwnedByYou":
+		err = BucketAlreadyOwnedByYou{}
+	case "BucketNotEmpty":
+		err = BucketNotEmpty{}
+	case "NoSuchBucketPolicy":
+		err = PolicyNotFound{}
+	case "InvalidBucketName":
+		err = BucketNameInvalid{Bucket: bucket}
+	case "NoSuchBucket":
+		err = BucketNotFound{Bucket: bucket}
+	case "NoSuchKey":
+		if object != "" {
+			err = ObjectNotFound{Bucket: bucket, Object: object}
+		} else {
+			err = BucketNotFound{Bucket: bucket}
+		}
+	case "XMinioInvalidObjectName":
+		err = ObjectNameInvalid{}
+	case "AccessDenied":
+		err = PrefixAccessDenied{
+			Bucket: bucket,
+			Object: object,
+		}
+	case "XAmzContentSHA256Mismatch":
+		err = hash.SHA256Mismatch{}
+	case "NoSuchUpload":
+		err = InvalidUploadID{}
+	case "EntityTooSmall":
+		err = PartTooSmall{}
+	}
+
+	e.Cause = err
+	return e
+}

cmd/gateway-handlers.go

@@ -18,7 +18,7 @@ package cmd
 
 import (
 	"io"
-	"io/ioutil"
+	goioutil "io/ioutil"
 	"net"
 	"net/http"
 	"strconv"
@@ -29,6 +29,7 @@ import (
 	router "github.com/gorilla/mux"
 	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/ioutil"
 )
 
 // GetObjectHandler - GET Object
@@ -60,7 +61,7 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
 			return
 		}
 	case authTypeSigned, authTypePresigned:
-		s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
+		s3Error := isReqAuthenticated(r, globalServerConfig.GetRegion())
 		if s3Error != ErrNone {
 			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
@@ -118,32 +119,19 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
 		startOffset = hrange.offsetBegin
 		length = hrange.getLength()
 	}
-	// Indicates if any data was written to the http.ResponseWriter
-	dataWritten := false
-	// io.Writer type which keeps track if any data was written.
-	writer := funcToWriter(func(p []byte) (int, error) {
-		if !dataWritten {
-			// Set headers on the first write.
-			// Set standard object headers.
-			setObjectHeaders(w, objInfo, hrange)
-
-			// Set any additional requested response headers.
-			setHeadGetRespHeaders(w, r.URL.Query())
-
-			dataWritten = true
-		}
-		return w.Write(p)
-	})
 
 	getObject := objectAPI.GetObject
 	if reqAuthType == authTypeAnonymous {
 		getObject = objectAPI.AnonGetObject
 	}
 
+	setObjectHeaders(w, objInfo, hrange)
+	setHeadGetRespHeaders(w, r.URL.Query())
+	httpWriter := ioutil.WriteOnClose(w)
 	// Reads the object at startOffset and writes to mw.
-	if err = getObject(bucket, object, startOffset, length, writer); err != nil {
+	if err = getObject(bucket, object, startOffset, length, httpWriter); err != nil {
 		errorIf(err, "Unable to write to client.")
-		if !dataWritten {
+		if !httpWriter.HasWritten() {
 			// Error response only if no data has been written to client yet. i.e if
 			// partial data has already been written before an error
 			// occurred then no point in setting StatusCode and
@@ -152,11 +140,11 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 		return
 	}
-	if !dataWritten {
-		// If ObjectAPI.GetObject did not return error and no data has
-		// been written it would mean that it is a 0-byte object.
-		// call wrter.Write(nil) to set appropriate headers.
-		writer.Write(nil)
+	if err = httpWriter.Close(); err != nil {
+		if !httpWriter.HasWritten() { // write error response only if no data has been written to client yet
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	// Get host and port from Request.RemoteAddr.
@@ -198,6 +186,14 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
 	bucket = vars["bucket"]
 	object = vars["object"]
 
+	// Validate storage class metadata if present
+	if _, ok := r.Header[amzStorageClassCanonical]; ok {
+		if !isValidStorageClassMeta(r.Header.Get(amzStorageClassCanonical)) {
+			writeErrorResponse(w, ErrInvalidStorageClass, r.URL)
+			return
+		}
+	}
+
 	// TODO: we should validate the object name here
 
 	// Get Content-Md5 sent by client and verify if valid
@@ -298,7 +294,7 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
 			return
 		}
 	case authTypePresigned, authTypeSigned:
-		if s3Error := reqSignatureV4Verify(r, serverConfig.GetRegion()); s3Error != ErrNone {
+		if s3Error := reqSignatureV4Verify(r, globalServerConfig.GetRegion()); s3Error != ErrNone {
 			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
@@ -369,7 +365,7 @@ func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.R
 			return
 		}
 	case authTypeSigned, authTypePresigned:
-		s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
+		s3Error := isReqAuthenticated(r, globalServerConfig.GetRegion())
 		if s3Error != ErrNone {
 			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
@@ -441,7 +437,7 @@ func (api gatewayAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -472,7 +468,7 @@ func (api gatewayAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *h
 	// Read access policy up to maxAccessPolicySize.
 	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 	// bucket policies are limited to 20KB in size, using a limit reader.
-	policyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	policyBytes, err := goioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
 	if err != nil {
 		errorIf(err, "Unable to read from client.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -504,7 +500,7 @@ func (api gatewayAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -538,7 +534,7 @@ func (api gatewayAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -608,7 +604,7 @@ func (api gatewayAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	// PutBucket does not have any bucket action.
-	s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion())
+	s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
 	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
@@ -655,7 +651,7 @@ func (api gatewayAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http
 	}
 
 	// DeleteBucket does not have any bucket action.
-	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -702,7 +698,7 @@ func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *htt
 			return
 		}
 	case authTypeSigned, authTypePresigned:
-		s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
+		s3Error := isReqAuthenticated(r, globalServerConfig.GetRegion())
 		if s3Error != ErrNone {
 			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
@@ -776,7 +772,7 @@ func (api gatewayAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *htt
 			return
 		}
 	case authTypeSigned, authTypePresigned:
-		s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
+		s3Error := isReqAuthenticated(r, globalServerConfig.GetRegion())
 		if s3Error != ErrNone {
 			errorIf(errSignatureMismatch, dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
@@ -855,7 +851,7 @@ func (api gatewayAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.R
 			return
 		}
 	case authTypeSigned, authTypePresigned:
-		s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
+		s3Error := isReqAuthenticated(r, globalServerConfig.GetRegion())
 		if s3Error != ErrNone {
 			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
@@ -910,7 +906,7 @@ func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r
 		s3Error := isReqAuthenticated(r, globalMinioDefaultRegion)
 		if s3Error == ErrInvalidRegion {
 			// Clients like boto3 send getBucketLocation() call signed with region that is configured.
-			s3Error = isReqAuthenticated(r, serverConfig.GetRegion())
+			s3Error = isReqAuthenticated(r, globalServerConfig.GetRegion())
 		}
 		if s3Error != ErrNone {
 			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
@@ -939,7 +935,7 @@ func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r
 	// Generate response.
 	encodedSuccessResponse := encodeResponse(LocationResponse{})
 	// Get current region.
-	region := serverConfig.GetRegion()
+	region := globalServerConfig.GetRegion()
 	if region != globalMinioDefaultRegion {
 		encodedSuccessResponse = encodeResponse(LocationResponse{
 			Location: region,

cmd/gateway-main.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"errors"
 	"fmt"
 	"net/url"
 	"os"
@@ -28,219 +27,28 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/cli"
+	"github.com/minio/minio/pkg/errors"
 	miniohttp "github.com/minio/minio/pkg/http"
 )
 
-const azureGatewayTemplate = `NAME:
-  {{.HelpName}} - {{.Usage}}
-
-USAGE:
-  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
-{{if .VisibleFlags}}
-FLAGS:
-  {{range .VisibleFlags}}{{.}}
-  {{end}}{{end}}
-ENDPOINT:
-  Azure server endpoint. Default ENDPOINT is https://core.windows.net
-
-ENVIRONMENT VARIABLES:
-  ACCESS:
-     MINIO_ACCESS_KEY: Username or access key of Azure storage.
-     MINIO_SECRET_KEY: Password or secret key of Azure storage.
-
-  BROWSER:
-     MINIO_BROWSER: To disable web browser access, set this value to "off".
-
-EXAMPLES:
-  1. Start minio gateway server for Azure Blob Storage backend.
-      $ export MINIO_ACCESS_KEY=azureaccountname
-      $ export MINIO_SECRET_KEY=azureaccountkey
-      $ {{.HelpName}}
-
-  2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
-      $ export MINIO_ACCESS_KEY=azureaccountname
-      $ export MINIO_SECRET_KEY=azureaccountkey
-      $ {{.HelpName}} https://azure.example.com
-`
-
-const s3GatewayTemplate = `NAME:
-  {{.HelpName}} - {{.Usage}}
-
-USAGE:
-  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
-{{if .VisibleFlags}}
-FLAGS:
-  {{range .VisibleFlags}}{{.}}
-  {{end}}{{end}}
-ENDPOINT:
-  S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com
-
-ENVIRONMENT VARIABLES:
-  ACCESS:
-     MINIO_ACCESS_KEY: Username or access key of S3 storage.
-     MINIO_SECRET_KEY: Password or secret key of S3 storage.
-
-  BROWSER:
-     MINIO_BROWSER: To disable web browser access, set this value to "off".
-
-EXAMPLES:
-  1. Start minio gateway server for AWS S3 backend.
-      $ export MINIO_ACCESS_KEY=accesskey
-      $ export MINIO_SECRET_KEY=secretkey
-      $ {{.HelpName}}
-
-  2. Start minio gateway server for S3 backend on custom endpoint.
-      $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
-      $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
-      $ {{.HelpName}} https://play.minio.io:9000
-`
-
-const gcsGatewayTemplate = `NAME:
-  {{.HelpName}} - {{.Usage}}
-
-USAGE:
-  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [PROJECTID]
-{{if .VisibleFlags}}
-FLAGS:
-  {{range .VisibleFlags}}{{.}}
-  {{end}}{{end}}
-PROJECTID:
-  GCS project-id should be provided if GOOGLE_APPLICATION_CREDENTIALS environmental variable is not set.
-
-ENVIRONMENT VARIABLES:
-  ACCESS:
-     MINIO_ACCESS_KEY: Username or access key of GCS.
-     MINIO_SECRET_KEY: Password or secret key of GCS.
-
-  BROWSER:
-     MINIO_BROWSER: To disable web browser access, set this value to "off".
-
-  GCS credentials file:
-     GOOGLE_APPLICATION_CREDENTIALS: Path to credentials.json
-
-EXAMPLES:
-  1. Start minio gateway server for GCS backend.
-      $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
-      (Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials)
-      $ export MINIO_ACCESS_KEY=accesskey
-      $ export MINIO_SECRET_KEY=secretkey
-      $ {{.HelpName}} mygcsprojectid
-
-`
-
-const b2GatewayTemplate = `NAME:
-  {{.HelpName}} - {{.Usage}}
-
-USAGE:
-  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}}
-{{if .VisibleFlags}}
-FLAGS:
-  {{range .VisibleFlags}}{{.}}
-  {{end}}{{end}}
-ENVIRONMENT VARIABLES:
-  ACCESS:
-     MINIO_ACCESS_KEY: B2 account id.
-     MINIO_SECRET_KEY: B2 application key.
-
-  BROWSER:
-     MINIO_BROWSER: To disable web browser access, set this value to "off".
-
-EXAMPLES:
-  1. Start minio gateway server for B2 backend.
-      $ export MINIO_ACCESS_KEY=accountID
-      $ export MINIO_SECRET_KEY=applicationKey
-      $ {{.HelpName}}
-
-`
-
 var (
-	azureBackendCmd = cli.Command{
-		Name:               "azure",
-		Usage:              "Microsoft Azure Blob Storage.",
-		Action:             azureGatewayMain,
-		CustomHelpTemplate: azureGatewayTemplate,
-		Flags:              append(serverFlags, globalFlags...),
-		HideHelpCommand:    true,
-	}
-
-	s3BackendCmd = cli.Command{
-		Name:               "s3",
-		Usage:              "Amazon Simple Storage Service (S3).",
-		Action:             s3GatewayMain,
-		CustomHelpTemplate: s3GatewayTemplate,
-		Flags:              append(serverFlags, globalFlags...),
-		HideHelpCommand:    true,
-	}
-
-	gcsBackendCmd = cli.Command{
-		Name:               "gcs",
-		Usage:              "Google Cloud Storage.",
-		Action:             gcsGatewayMain,
-		CustomHelpTemplate: gcsGatewayTemplate,
-		Flags:              append(serverFlags, globalFlags...),
-		HideHelpCommand:    true,
-	}
-
-	b2BackendCmd = cli.Command{
-		Name:               "b2",
-		Usage:              "Backblaze B2.",
-		Action:             b2GatewayMain,
-		CustomHelpTemplate: b2GatewayTemplate,
-		Flags:              append(serverFlags, globalFlags...),
-		HideHelpCommand:    true,
-	}
-
 	gatewayCmd = cli.Command{
 		Name:            "gateway",
 		Usage:           "Start object storage gateway.",
 		Flags:           append(serverFlags, globalFlags...),
 		HideHelpCommand: true,
-		Subcommands:     []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd, b2BackendCmd},
 	}
 )
 
-// Represents the type of the gateway backend.
-type gatewayBackend string
-
-const (
-	azureBackend gatewayBackend = "azure"
-	s3Backend    gatewayBackend = "s3"
-	gcsBackend   gatewayBackend = "gcs"
-	b2Backend    gatewayBackend = "b2"
-	// Add more backends here.
-)
-
-// Initialize gateway layer depending on the backend type.
-// Supported backend types are
-//
-// - Azure Blob Storage.
-// - AWS S3.
-// - Google Cloud Storage.
-// - Backblaze B2.
-// - Add your favorite backend here.
-func newGatewayLayer(backendType gatewayBackend, arg string) (GatewayLayer, error) {
-	switch backendType {
-	case azureBackend:
-		return newAzureLayer(arg)
-	case s3Backend:
-		return newS3Gateway(arg)
-	case gcsBackend:
-		// FIXME: The following print command is temporary and
-		// will be removed when gcs is ready for production use.
-		log.Println(colorYellow("\n               *** Warning: Not Ready for Production ***"))
-		return newGCSGateway(arg)
-	case b2Backend:
-		// FIXME: The following print command is temporary and
-		// will be removed when B2 is ready for production use.
-		log.Println(colorYellow("\n               *** Warning: Not Ready for Production ***"))
-		return newB2Gateway()
-	}
-
-	return nil, fmt.Errorf("Unrecognized backend type %s", backendType)
+// RegisterGatewayCommand registers a new command for gateway.
+func RegisterGatewayCommand(cmd cli.Command) error {
+	cmd.Flags = append(append(cmd.Flags, append(cmd.Flags, serverFlags...)...), globalFlags...)
+	gatewayCmd.Subcommands = append(gatewayCmd.Subcommands, cmd)
+	return nil
 }
 
-// Return endpoint.
-func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error) {
+// ParseGatewayEndpoint - Return endpoint.
+func ParseGatewayEndpoint(arg string) (endPoint string, secure bool, err error) {
 	schemeSpecified := len(strings.Split(arg, "://")) > 1
 	if !schemeSpecified {
 		// Default connection will be "secure".
@@ -262,8 +70,8 @@ func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error)
 	}
 }
 
-// Validate gateway arguments.
-func validateGatewayArguments(serverAddr, endpointAddr string) error {
+// ValidateGatewayArguments - Validate gateway arguments.
+func ValidateGatewayArguments(serverAddr, endpointAddr string) error {
 	if err := CheckLocalServerAddr(serverAddr); err != nil {
 		return err
 	}
@@ -286,68 +94,24 @@ func validateGatewayArguments(serverAddr, endpointAddr string) error {
 			return err
 		}
 		if sameTarget {
-			return errors.New("endpoint points to the local gateway")
+			return fmt.Errorf("endpoint points to the local gateway")
 		}
 	}
 	return nil
 }
 
-// Handler for 'minio gateway azure' command line.
-func azureGatewayMain(ctx *cli.Context) {
-	if ctx.Args().Present() && ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, "azure", 1)
+// StartGateway - handler for 'minio gateway <name>'.
+func StartGateway(ctx *cli.Context, gw Gateway) {
+	if gw == nil {
+		fatalIf(errUnexpected, "Gateway implementation not initialized, exiting.")
 	}
 
-	// Validate gateway arguments.
-	fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument")
-
-	gatewayMain(ctx, azureBackend)
-}
-
-// Handler for 'minio gateway s3' command line.
-func s3GatewayMain(ctx *cli.Context) {
-	if ctx.Args().Present() && ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, "s3", 1)
+	// Validate if we have access, secret set through environment.
+	gatewayName := gw.Name()
+	if ctx.Args().First() == "help" {
+		cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
 	}
 
-	// Validate gateway arguments.
-	fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument")
-
-	gatewayMain(ctx, s3Backend)
-}
-
-// Handler for 'minio gateway gcs' command line
-func gcsGatewayMain(ctx *cli.Context) {
-	if ctx.Args().Present() && ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
-	}
-
-	projectID := ctx.Args().First()
-	if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
-		errorIf(errGCSProjectIDNotFound, "project-id should be provided as argument or GOOGLE_APPLICATION_CREDENTIALS should be set with path to credentials.json")
-		cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
-	}
-	if projectID != "" && !isValidGCSProjectIDFormat(projectID) {
-		errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First())
-		cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
-	}
-
-	gatewayMain(ctx, gcsBackend)
-}
-
-func b2GatewayMain(ctx *cli.Context) {
-	if ctx.Args().Present() && ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, "b2", 1)
-	}
-
-	// Validate gateway arguments.
-	fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument")
-
-	gatewayMain(ctx, b2Backend)
-}
-
-// Handler for 'minio gateway'.
-func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
 	// Get quiet flag from command line argument.
 	quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
 	if quietFlag {
@@ -368,7 +132,8 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
 
 	// Validate if we have access, secret set through environment.
 	if !globalIsEnvCreds {
-		fatalIf(fmt.Errorf("Access and Secret keys should be set through ENVs for backend [%s]", backendType), "")
+		errorIf(fmt.Errorf("Access and secret keys not set"), "Access and Secret keys should be set through ENVs for backend [%s]", gatewayName)
+		cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
 	}
 
 	// Create certs path.
@@ -377,20 +142,20 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
 	// Initialize gateway config.
 	initConfig()
 
-	// Enable loggers as per configuration file.
-	enableLoggers()
-
 	// Init the error tracing module.
-	initError()
+	errors.Init(GOPATH, "github.com/minio/minio")
 
 	// Check and load SSL certificates.
 	var err error
 	globalPublicCerts, globalRootCAs, globalTLSCertificate, globalIsSSL, err = getSSLConfig()
 	fatalIf(err, "Invalid SSL certificate file")
 
+	// Set system resources to maximum.
+	errorIf(setMaxResources(), "Unable to change resource limit")
+
 	initNSLock(false) // Enable local namespace lock.
 
-	newObject, err := newGatewayLayer(backendType, ctx.Args().First())
+	newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential())
 	fatalIf(err, "Unable to initialize gateway layer")
 
 	router := mux.NewRouter().SkipClean(true)
@@ -445,23 +210,17 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
 
 	// Prints the formatted startup message once object layer is initialized.
 	if !quietFlag {
-		mode := ""
-		switch gatewayBackend(backendType) {
-		case azureBackend:
-			mode = globalMinioModeGatewayAzure
-		case gcsBackend:
-			mode = globalMinioModeGatewayGCS
-		case s3Backend:
-			mode = globalMinioModeGatewayS3
-		case b2Backend:
-			mode = globalMinioModeGatewayB2
-		}
-
+		mode := globalMinioModeGatewayPrefix + gatewayName
 		// Check update mode.
 		checkUpdate(mode)
 
+		// Print a warning message if gateway is not ready for production before the startup banner.
+		if !gw.Production() {
+			log.Println(colorYellow("\n               *** Warning: Not Ready for Production ***"))
+		}
+
 		// Print gateway startup message.
-		printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), backendType)
+		printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
 	}
 
 	handleSignals()

cmd/gateway-main_test.go

@@ -19,8 +19,21 @@ package cmd
 import (
 	"strings"
 	"testing"
+
+	"github.com/minio/cli"
 )
 
+// Test RegisterGatewayCommand
+func TestRegisterGatewayCommand(t *testing.T) {
+	var err error
+
+	cmd := cli.Command{Name: "test"}
+	err = RegisterGatewayCommand(cmd)
+	if err != nil {
+		t.Errorf("RegisterGatewayCommand got unexpected error: %s", err)
+	}
+}
+
 // Test parseGatewayEndpoint
 func TestParseGatewayEndpoint(t *testing.T) {
 	testCases := []struct {
@@ -39,7 +52,7 @@ func TestParseGatewayEndpoint(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		endPoint, secure, err := parseGatewayEndpoint(test.arg)
+		endPoint, secure, err := ParseGatewayEndpoint(test.arg)
 		errReturned := err != nil
 
 		if endPoint != test.endPoint ||
@@ -74,7 +87,7 @@ func TestValidateGatewayArguments(t *testing.T) {
 		{":9000", nonLoopBackIP + ":9000", false},
 	}
 	for i, test := range testCases {
-		err := validateGatewayArguments(test.serverAddr, test.endpointAddr)
+		err := ValidateGatewayArguments(test.serverAddr, test.endpointAddr)
 		if test.valid && err != nil {
 			t.Errorf("Test %d expected not to return error but got %s", i+1, err)
 		}

cmd/gateway-router.go

@@ -18,13 +18,30 @@ package cmd
 
 import (
 	"io"
+	"net/http"
 
 	router "github.com/gorilla/mux"
 	"github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
 )
 
-// GatewayLayer - Interface to implement gateway mode.
+// GatewayMinioSysTmp prefix is used in Azure/GCS gateway for save metadata sent by Initialize Multipart Upload API.
+const GatewayMinioSysTmp = "minio.sys.tmp/"
+
+// Gateway represents a gateway backend.
+type Gateway interface {
+	// Name returns the unique name of the gateway.
+	Name() string
+
+	// NewGatewayLayer returns a new gateway layer.
+	NewGatewayLayer(creds auth.Credentials) (GatewayLayer, error)
+
+	// Returns true if gateway is ready for production.
+	Production() bool
+}
+
+// GatewayLayer - interface to implement gateway mode.
 type GatewayLayer interface {
 	ObjectLayer
 
@@ -62,69 +79,78 @@ func registerGatewayAPIRouter(mux *router.Router, gw GatewayLayer) {
 	// API Router
 	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
 
-	// Bucket router
-	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
+	var routers []*router.Router
+	if globalDomainName != "" {
+		routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
+	}
+	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
-	/// Object operations
+	// Object operations
+	for _, bucket := range routers {
+		/// Object operations
 
-	// HeadObject
-	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
-	// CopyObjectPart
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-	// PutObjectPart
-	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-	// ListObjectPxarts
-	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
-	// CompleteMultipartUpload
-	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
-	// NewMultipartUpload
-	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
-	// AbortMultipartUpload
-	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
-	// GetObject
-	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
-	// CopyObject
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
-	// PutObject
-	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
-	// DeleteObject
-	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
+		// HeadObject
+		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
+		// CopyObjectPart
+		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		// PutObjectPart
+		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		// ListObjectPxarts
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
+		// CompleteMultipartUpload
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// NewMultipartUpload
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
+		// AbortMultipartUpload
+		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// GetObject
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
+		// CopyObject
+		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
+		// PutObject
+		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
+		// DeleteObject
+		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
 
-	/// Bucket operations
+		/// Bucket operations
 
-	// GetBucketLocation
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
-	// GetBucketPolicy
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
-	// GetBucketNotification
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
-	// ListenBucketNotification
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
-	// ListMultipartUploads
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
-	// ListObjectsV2
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
-	// ListObjectsV1 (Legacy)
-	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
-	// PutBucketPolicy
-	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
-	// PutBucketNotification
-	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
-	// PutBucket
-	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
-	// HeadBucket
-	bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
-	// PostPolicy
-	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
-	// DeleteMultipleObjects
-	bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler))
-	// DeleteBucketPolicy
-	bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
-	// DeleteBucket
-	bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+		// GetBucketLocation
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
+		// GetBucketPolicy
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+		// GetBucketNotification
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
+		// ListenBucketNotification
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
+		// ListMultipartUploads
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
+		// ListObjectsV2
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
+		// ListObjectsV1 (Legacy)
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
+		// PutBucketPolicy
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
+		// PutBucketNotification
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
+		// PutBucket
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
+		// HeadBucket
+		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
+		// PostPolicy
+		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
+		// DeleteMultipleObjects
+		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
+		// DeleteBucketPolicy
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
+		// DeleteBucket
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+	}
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods("GET").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+	apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+
+	// If none of the routes match.
+	apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
 }

cmd/gateway-s3.go

@@ -1,566 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"io"
-	"net/http"
-
-	minio "github.com/minio/minio-go"
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/s3utils"
-	"github.com/minio/minio/pkg/hash"
-)
-
-// s3ToObjectError converts Minio errors to minio object layer errors.
-func s3ToObjectError(err error, params ...string) error {
-	if err == nil {
-		return nil
-	}
-
-	e, ok := err.(*Error)
-	if !ok {
-		// Code should be fixed if this function is called without doing traceError()
-		// Else handling different situations in this function makes this function complicated.
-		errorIf(err, "Expected type *Error")
-		return err
-	}
-
-	err = e.e
-
-	bucket := ""
-	object := ""
-	if len(params) >= 1 {
-		bucket = params[0]
-	}
-	if len(params) == 2 {
-		object = params[1]
-	}
-
-	minioErr, ok := err.(minio.ErrorResponse)
-	if !ok {
-		// We don't interpret non Minio errors. As minio errors will
-		// have StatusCode to help to convert to object errors.
-		return e
-	}
-
-	switch minioErr.Code {
-	case "BucketAlreadyOwnedByYou":
-		err = BucketAlreadyOwnedByYou{}
-	case "BucketNotEmpty":
-		err = BucketNotEmpty{}
-	case "NoSuchBucketPolicy":
-		err = PolicyNotFound{}
-	case "InvalidBucketName":
-		err = BucketNameInvalid{Bucket: bucket}
-	case "NoSuchBucket":
-		err = BucketNotFound{Bucket: bucket}
-	case "NoSuchKey":
-		if object != "" {
-			err = ObjectNotFound{Bucket: bucket, Object: object}
-		} else {
-			err = BucketNotFound{Bucket: bucket}
-		}
-	case "XMinioInvalidObjectName":
-		err = ObjectNameInvalid{}
-	case "AccessDenied":
-		err = PrefixAccessDenied{
-			Bucket: bucket,
-			Object: object,
-		}
-	case "XAmzContentSHA256Mismatch":
-		err = hash.SHA256Mismatch{}
-	case "NoSuchUpload":
-		err = InvalidUploadID{}
-	case "EntityTooSmall":
-		err = PartTooSmall{}
-	}
-
-	e.e = err
-	return e
-}
-
-// s3Objects implements gateway for Minio and S3 compatible object storage servers.
-type s3Objects struct {
-	gatewayUnsupported
-	Client     *minio.Core
-	anonClient *minio.Core
-}
-
-// newS3Gateway returns s3 gatewaylayer
-func newS3Gateway(host string) (GatewayLayer, error) {
-	var err error
-	var endpoint string
-	var secure = true
-
-	// Validate host parameters.
-	if host != "" {
-		// Override default params if the host is provided
-		endpoint, secure, err = parseGatewayEndpoint(host)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Default endpoint parameters
-	if endpoint == "" {
-		endpoint = "s3.amazonaws.com"
-	}
-
-	creds := serverConfig.GetCredential()
-
-	// Initialize minio client object.
-	client, err := minio.NewCore(endpoint, creds.AccessKey, creds.SecretKey, secure)
-	if err != nil {
-		return nil, err
-	}
-
-	anonClient, err := minio.NewCore(endpoint, "", "", secure)
-	if err != nil {
-		return nil, err
-	}
-	anonClient.SetCustomTransport(newCustomHTTPTransport())
-
-	return &s3Objects{
-		Client:     client,
-		anonClient: anonClient,
-	}, nil
-}
-
-// Shutdown saves any gateway metadata to disk
-// if necessary and reload upon next restart.
-func (l *s3Objects) Shutdown() error {
-	// TODO
-	return nil
-}
-
-// StorageInfo is not relevant to S3 backend.
-func (l *s3Objects) StorageInfo() (si StorageInfo) {
-	return si
-}
-
-// MakeBucket creates a new container on S3 backend.
-func (l *s3Objects) MakeBucketWithLocation(bucket, location string) error {
-	err := l.Client.MakeBucket(bucket, location)
-	if err != nil {
-		return s3ToObjectError(traceError(err), bucket)
-	}
-	return err
-}
-
-// GetBucketInfo gets bucket metadata..
-func (l *s3Objects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
-	// Verify if bucket name is valid.
-	// We are using a separate helper function here to validate bucket
-	// names instead of IsValidBucketName() because there is a possibility
-	// that certains users might have buckets which are non-DNS compliant
-	// in us-east-1 and we might severely restrict them by not allowing
-	// access to these buckets.
-	// Ref - http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
-	if s3utils.CheckValidBucketName(bucket) != nil {
-		return bi, traceError(BucketNameInvalid{Bucket: bucket})
-	}
-
-	buckets, err := l.Client.ListBuckets()
-	if err != nil {
-		return bi, s3ToObjectError(traceError(err), bucket)
-	}
-
-	for _, bi := range buckets {
-		if bi.Name != bucket {
-			continue
-		}
-
-		return BucketInfo{
-			Name:    bi.Name,
-			Created: bi.CreationDate,
-		}, nil
-	}
-
-	return bi, traceError(BucketNotFound{Bucket: bucket})
-}
-
-// ListBuckets lists all S3 buckets
-func (l *s3Objects) ListBuckets() ([]BucketInfo, error) {
-	buckets, err := l.Client.ListBuckets()
-	if err != nil {
-		return nil, s3ToObjectError(traceError(err))
-	}
-
-	b := make([]BucketInfo, len(buckets))
-	for i, bi := range buckets {
-		b[i] = BucketInfo{
-			Name:    bi.Name,
-			Created: bi.CreationDate,
-		}
-	}
-
-	return b, err
-}
-
-// DeleteBucket deletes a bucket on S3
-func (l *s3Objects) DeleteBucket(bucket string) error {
-	err := l.Client.RemoveBucket(bucket)
-	if err != nil {
-		return s3ToObjectError(traceError(err), bucket)
-	}
-	return nil
-}
-
-// ListObjects lists all blobs in S3 bucket filtered by prefix
-func (l *s3Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	result, err := l.Client.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
-	if err != nil {
-		return loi, s3ToObjectError(traceError(err), bucket)
-	}
-
-	return fromMinioClientListBucketResult(bucket, result), nil
-}
-
-// ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
-func (l *s3Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi ListObjectsV2Info, e error) {
-	result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
-	if err != nil {
-		return loi, s3ToObjectError(traceError(err), bucket)
-	}
-
-	return fromMinioClientListBucketV2Result(bucket, result), nil
-}
-
-// fromMinioClientListBucketV2Result converts minio ListBucketResult to ListObjectsInfo
-func fromMinioClientListBucketV2Result(bucket string, result minio.ListBucketV2Result) ListObjectsV2Info {
-	objects := make([]ObjectInfo, len(result.Contents))
-
-	for i, oi := range result.Contents {
-		objects[i] = fromMinioClientObjectInfo(bucket, oi)
-	}
-
-	prefixes := make([]string, len(result.CommonPrefixes))
-	for i, p := range result.CommonPrefixes {
-		prefixes[i] = p.Prefix
-	}
-
-	return ListObjectsV2Info{
-		IsTruncated: result.IsTruncated,
-		Prefixes:    prefixes,
-		Objects:     objects,
-
-		ContinuationToken:     result.ContinuationToken,
-		NextContinuationToken: result.NextContinuationToken,
-	}
-}
-
-// fromMinioClientListBucketResult converts minio ListBucketResult to ListObjectsInfo
-func fromMinioClientListBucketResult(bucket string, result minio.ListBucketResult) ListObjectsInfo {
-	objects := make([]ObjectInfo, len(result.Contents))
-
-	for i, oi := range result.Contents {
-		objects[i] = fromMinioClientObjectInfo(bucket, oi)
-	}
-
-	prefixes := make([]string, len(result.CommonPrefixes))
-	for i, p := range result.CommonPrefixes {
-		prefixes[i] = p.Prefix
-	}
-
-	return ListObjectsInfo{
-		IsTruncated: result.IsTruncated,
-		NextMarker:  result.NextMarker,
-		Prefixes:    prefixes,
-		Objects:     objects,
-	}
-}
-
-// GetObject reads an object from S3. Supports additional
-// parameters like offset and length which are synonymous with
-// HTTP Range requests.
-//
-// startOffset indicates the starting read location of the object.
-// length indicates the total length of the object.
-func (l *s3Objects) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
-	if length < 0 && length != -1 {
-		return s3ToObjectError(traceError(errInvalidArgument), bucket, key)
-	}
-
-	opts := minio.GetObjectOptions{}
-	if startOffset >= 0 && length >= 0 {
-		if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
-			return s3ToObjectError(traceError(err), bucket, key)
-		}
-	}
-	object, _, err := l.Client.GetObject(bucket, key, opts)
-	if err != nil {
-		return s3ToObjectError(traceError(err), bucket, key)
-	}
-	defer object.Close()
-
-	if _, err := io.Copy(writer, object); err != nil {
-		return s3ToObjectError(traceError(err), bucket, key)
-	}
-	return nil
-}
-
-// fromMinioClientObjectInfo converts minio ObjectInfo to gateway ObjectInfo
-func fromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
-	userDefined := fromMinioClientMetadata(oi.Metadata)
-	userDefined["Content-Type"] = oi.ContentType
-
-	return ObjectInfo{
-		Bucket:          bucket,
-		Name:            oi.Key,
-		ModTime:         oi.LastModified,
-		Size:            oi.Size,
-		ETag:            canonicalizeETag(oi.ETag),
-		UserDefined:     userDefined,
-		ContentType:     oi.ContentType,
-		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
-	}
-}
-
-// GetObjectInfo reads object info and replies back ObjectInfo
-func (l *s3Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
-	oi, err := l.Client.StatObject(bucket, object, minio.StatObjectOptions{})
-	if err != nil {
-		return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
-	}
-
-	return fromMinioClientObjectInfo(bucket, oi), nil
-}
-
-// PutObject creates a new object with the incoming data,
-func (l *s3Objects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
-	oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5(), data.SHA256(), toMinioClientMetadata(metadata))
-	if err != nil {
-		return objInfo, s3ToObjectError(traceError(err), bucket, object)
-	}
-
-	return fromMinioClientObjectInfo(bucket, oi), nil
-}
-
-// CopyObject copies an object from source bucket to a destination bucket.
-func (l *s3Objects) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
-	// Set this header such that following CopyObject() always sets the right metadata on the destination.
-	// metadata input is already a trickled down value from interpreting x-amz-metadata-directive at
-	// handler layer. So what we have right now is supposed to be applied on the destination object anyways.
-	// So preserve it by adding "REPLACE" directive to save all the metadata set by CopyObject API.
-	metadata["x-amz-metadata-directive"] = "REPLACE"
-	if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, metadata); err != nil {
-		return objInfo, s3ToObjectError(traceError(err), srcBucket, srcObject)
-	}
-	return l.GetObjectInfo(dstBucket, dstObject)
-}
-
-// DeleteObject deletes a blob in bucket
-func (l *s3Objects) DeleteObject(bucket string, object string) error {
-	err := l.Client.RemoveObject(bucket, object)
-	if err != nil {
-		return s3ToObjectError(traceError(err), bucket, object)
-	}
-
-	return nil
-}
-
-// fromMinioClientUploadMetadata converts ObjectMultipartInfo to uploadMetadata
-func fromMinioClientUploadMetadata(omi minio.ObjectMultipartInfo) uploadMetadata {
-	return uploadMetadata{
-		Object:    omi.Key,
-		UploadID:  omi.UploadID,
-		Initiated: omi.Initiated,
-	}
-}
-
-// fromMinioClientListMultipartsInfo converts minio ListMultipartUploadsResult to ListMultipartsInfo
-func fromMinioClientListMultipartsInfo(lmur minio.ListMultipartUploadsResult) ListMultipartsInfo {
-	uploads := make([]uploadMetadata, len(lmur.Uploads))
-
-	for i, um := range lmur.Uploads {
-		uploads[i] = fromMinioClientUploadMetadata(um)
-	}
-
-	commonPrefixes := make([]string, len(lmur.CommonPrefixes))
-	for i, cp := range lmur.CommonPrefixes {
-		commonPrefixes[i] = cp.Prefix
-	}
-
-	return ListMultipartsInfo{
-		KeyMarker:          lmur.KeyMarker,
-		UploadIDMarker:     lmur.UploadIDMarker,
-		NextKeyMarker:      lmur.NextKeyMarker,
-		NextUploadIDMarker: lmur.NextUploadIDMarker,
-		MaxUploads:         int(lmur.MaxUploads),
-		IsTruncated:        lmur.IsTruncated,
-		Uploads:            uploads,
-		Prefix:             lmur.Prefix,
-		Delimiter:          lmur.Delimiter,
-		CommonPrefixes:     commonPrefixes,
-		EncodingType:       lmur.EncodingType,
-	}
-
-}
-
-// ListMultipartUploads lists all multipart uploads.
-func (l *s3Objects) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
-	result, err := l.Client.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
-	if err != nil {
-		return lmi, err
-	}
-
-	return fromMinioClientListMultipartsInfo(result), nil
-}
-
-// fromMinioClientMetadata converts minio metadata to map[string]string
-func fromMinioClientMetadata(metadata map[string][]string) map[string]string {
-	mm := map[string]string{}
-	for k, v := range metadata {
-		mm[http.CanonicalHeaderKey(k)] = v[0]
-	}
-	return mm
-}
-
-// toMinioClientMetadata converts metadata to map[string][]string
-func toMinioClientMetadata(metadata map[string]string) map[string]string {
-	mm := map[string]string{}
-	for k, v := range metadata {
-		mm[http.CanonicalHeaderKey(k)] = v
-	}
-	return mm
-}
-
-// NewMultipartUpload upload object in multiple parts
-func (l *s3Objects) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) {
-	// Create PutObject options
-	opts := minio.PutObjectOptions{UserMetadata: metadata}
-	uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts)
-	if err != nil {
-		return uploadID, s3ToObjectError(traceError(err), bucket, object)
-	}
-	return uploadID, nil
-}
-
-// fromMinioClientObjectPart converts minio ObjectPart to PartInfo
-func fromMinioClientObjectPart(op minio.ObjectPart) PartInfo {
-	return PartInfo{
-		Size:         op.Size,
-		ETag:         canonicalizeETag(op.ETag),
-		LastModified: op.LastModified,
-		PartNumber:   op.PartNumber,
-	}
-}
-
-// PutObjectPart puts a part of object in bucket
-func (l *s3Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) {
-	info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5(), data.SHA256())
-	if err != nil {
-		return pi, s3ToObjectError(traceError(err), bucket, object)
-	}
-
-	return fromMinioClientObjectPart(info), nil
-}
-
-// fromMinioClientObjectParts converts minio ObjectPart to PartInfo
-func fromMinioClientObjectParts(parts []minio.ObjectPart) []PartInfo {
-	toParts := make([]PartInfo, len(parts))
-	for i, part := range parts {
-		toParts[i] = fromMinioClientObjectPart(part)
-	}
-	return toParts
-}
-
-// fromMinioClientListPartsInfo converts minio ListObjectPartsResult to ListPartsInfo
-func fromMinioClientListPartsInfo(lopr minio.ListObjectPartsResult) ListPartsInfo {
-	return ListPartsInfo{
-		UploadID:             lopr.UploadID,
-		Bucket:               lopr.Bucket,
-		Object:               lopr.Key,
-		StorageClass:         "",
-		PartNumberMarker:     lopr.PartNumberMarker,
-		NextPartNumberMarker: lopr.NextPartNumberMarker,
-		MaxParts:             lopr.MaxParts,
-		IsTruncated:          lopr.IsTruncated,
-		EncodingType:         lopr.EncodingType,
-		Parts:                fromMinioClientObjectParts(lopr.ObjectParts),
-	}
-}
-
-// ListObjectParts returns all object parts for specified object in specified bucket
-func (l *s3Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, e error) {
-	result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
-	if err != nil {
-		return lpi, err
-	}
-
-	return fromMinioClientListPartsInfo(result), nil
-}
-
-// AbortMultipartUpload aborts a ongoing multipart upload
-func (l *s3Objects) AbortMultipartUpload(bucket string, object string, uploadID string) error {
-	err := l.Client.AbortMultipartUpload(bucket, object, uploadID)
-	return s3ToObjectError(traceError(err), bucket, object)
-}
-
-// toMinioClientCompletePart converts completePart to minio CompletePart
-func toMinioClientCompletePart(part completePart) minio.CompletePart {
-	return minio.CompletePart{
-		ETag:       part.ETag,
-		PartNumber: part.PartNumber,
-	}
-}
-
-// toMinioClientCompleteParts converts []completePart to minio []CompletePart
-func toMinioClientCompleteParts(parts []completePart) []minio.CompletePart {
-	mparts := make([]minio.CompletePart, len(parts))
-	for i, part := range parts {
-		mparts[i] = toMinioClientCompletePart(part)
-	}
-	return mparts
-}
-
-// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
-func (l *s3Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []completePart) (oi ObjectInfo, e error) {
-	err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, toMinioClientCompleteParts(uploadedParts))
-	if err != nil {
-		return oi, s3ToObjectError(traceError(err), bucket, object)
-	}
-
-	return l.GetObjectInfo(bucket, object)
-}
-
-// SetBucketPolicies sets policy on bucket
-func (l *s3Objects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
-	if err := l.Client.PutBucketPolicy(bucket, policyInfo); err != nil {
-		return s3ToObjectError(traceError(err), bucket, "")
-	}
-
-	return nil
-}
-
-// GetBucketPolicies will get policy on bucket
-func (l *s3Objects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
-	policyInfo, err := l.Client.GetBucketPolicy(bucket)
-	if err != nil {
-		return policy.BucketAccessPolicy{}, s3ToObjectError(traceError(err), bucket, "")
-	}
-	return policyInfo, nil
-}
-
-// DeleteBucketPolicies deletes all policies on bucket
-func (l *s3Objects) DeleteBucketPolicies(bucket string) error {
-	if err := l.Client.PutBucketPolicy(bucket, policy.BucketAccessPolicy{}); err != nil {
-		return s3ToObjectError(traceError(err), bucket, "")
-	}
-	return nil
-}

cmd/gateway-startup-msg.go

@@ -22,7 +22,7 @@ import (
 )
 
 // Prints the formatted startup message.
-func printGatewayStartupMessage(apiEndPoints []string, backendType gatewayBackend) {
+func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
 	strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
 
 	// Prints credential.
@@ -45,7 +45,7 @@ func printGatewayStartupMessage(apiEndPoints []string, backendType gatewayBacken
 // Prints common server startup message. Prints credential, region and browser access.
 func printGatewayCommonMsg(apiEndpoints []string) {
 	// Get saved credentials.
-	cred := serverConfig.GetCredential()
+	cred := globalServerConfig.GetCredential()
 
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 	// Colorize the message and print.

cmd/gateway-unsupported.go

@@ -17,62 +17,132 @@
 package cmd
 
 import (
+	"io"
+
+	"github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
 )
 
-type gatewayUnsupported struct{}
+// GatewayUnsupported list of unsupported call stubs for gateway.
+type GatewayUnsupported struct{}
 
-// CopyObjectPart - Not implemented.
-func (a gatewayUnsupported) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string,
-	partID int, startOffset int64, length int64) (info PartInfo, err error) {
-	return info, traceError(NotImplemented{})
+// ListMultipartUploads lists all multipart uploads.
+func (a GatewayUnsupported) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
+	return lmi, errors.Trace(NotImplemented{})
 }
 
-// HealBucket - Not relevant.
-func (a gatewayUnsupported) HealBucket(bucket string) error {
-	return traceError(NotImplemented{})
+// NewMultipartUpload upload object in multiple parts
+func (a GatewayUnsupported) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) {
+	return "", errors.Trace(NotImplemented{})
 }
 
-// ListBucketsHeal - Not relevant.
-func (a gatewayUnsupported) ListBucketsHeal() (buckets []BucketInfo, err error) {
-	return nil, traceError(NotImplemented{})
+// CopyObjectPart copy part of object to other bucket and object
+func (a GatewayUnsupported) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string, uploadID string, partID int, startOffset int64, length int64, metadata map[string]string) (pi PartInfo, err error) {
+	return pi, errors.Trace(NotImplemented{})
 }
 
-// HealObject - Not relevant.
-func (a gatewayUnsupported) HealObject(bucket, object string) (int, int, error) {
-	return 0, 0, traceError(NotImplemented{})
+// PutObjectPart puts a part of object in bucket
+func (a GatewayUnsupported) PutObjectPart(bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, err error) {
+	return pi, errors.Trace(NotImplemented{})
 }
 
-// ListObjectsHeal - Not relevant.
-func (a gatewayUnsupported) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	return loi, traceError(NotImplemented{})
+// ListObjectParts returns all object parts for specified object in specified bucket
+func (a GatewayUnsupported) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) {
+	return lpi, errors.Trace(NotImplemented{})
 }
 
-// ListUploadsHeal - Not relevant.
-func (a gatewayUnsupported) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
+// AbortMultipartUpload aborts a ongoing multipart upload
+func (a GatewayUnsupported) AbortMultipartUpload(bucket string, object string, uploadID string) error {
+	return errors.Trace(NotImplemented{})
+}
+
+// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
+func (a GatewayUnsupported) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, err error) {
+	return oi, errors.Trace(NotImplemented{})
+}
+
+// SetBucketPolicies sets policy on bucket
+func (a GatewayUnsupported) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
+	return errors.Trace(NotImplemented{})
+}
+
+// GetBucketPolicies will get policy on bucket
+func (a GatewayUnsupported) GetBucketPolicies(bucket string) (bal policy.BucketAccessPolicy, err error) {
+	return bal, errors.Trace(NotImplemented{})
+}
+
+// DeleteBucketPolicies deletes all policies on bucket
+func (a GatewayUnsupported) DeleteBucketPolicies(bucket string) error {
+	return errors.Trace(NotImplemented{})
+}
+
+// HealBucket - Not implemented stub
+func (a GatewayUnsupported) HealBucket(bucket string) error {
+	return errors.Trace(NotImplemented{})
+}
+
+// ListBucketsHeal - Not implemented stub
+func (a GatewayUnsupported) ListBucketsHeal() (buckets []BucketInfo, err error) {
+	return nil, errors.Trace(NotImplemented{})
+}
+
+// HealObject - Not implemented stub
+func (a GatewayUnsupported) HealObject(bucket, object string) (int, int, error) {
+	return 0, 0, errors.Trace(NotImplemented{})
+}
+
+// ListObjectsV2 - Not implemented stub
+func (a GatewayUnsupported) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
+	return result, errors.Trace(NotImplemented{})
+}
+
+// ListObjectsHeal - Not implemented stub
+func (a GatewayUnsupported) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
+	return loi, errors.Trace(NotImplemented{})
+}
+
+// ListUploadsHeal - Not implemented stub
+func (a GatewayUnsupported) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
 	delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
-	return lmi, traceError(NotImplemented{})
+	return lmi, errors.Trace(NotImplemented{})
 }
 
 // AnonListObjects - List objects anonymously
-func (a gatewayUnsupported) AnonListObjects(bucket string, prefix string, marker string, delimiter string,
+func (a GatewayUnsupported) AnonListObjects(bucket string, prefix string, marker string, delimiter string,
 	maxKeys int) (loi ListObjectsInfo, err error) {
-	return loi, traceError(NotImplemented{})
+	return loi, errors.Trace(NotImplemented{})
 }
 
 // AnonListObjectsV2 - List objects in V2 mode, anonymously
-func (a gatewayUnsupported) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int,
+func (a GatewayUnsupported) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int,
 	fetchOwner bool, startAfter string) (loi ListObjectsV2Info, err error) {
-	return loi, traceError(NotImplemented{})
+	return loi, errors.Trace(NotImplemented{})
 }
 
 // AnonGetBucketInfo - Get bucket metadata anonymously.
-func (a gatewayUnsupported) AnonGetBucketInfo(bucket string) (bi BucketInfo, err error) {
-	return bi, traceError(NotImplemented{})
+func (a GatewayUnsupported) AnonGetBucketInfo(bucket string) (bi BucketInfo, err error) {
+	return bi, errors.Trace(NotImplemented{})
 }
 
 // AnonPutObject creates a new object anonymously with the incoming data,
-func (a gatewayUnsupported) AnonPutObject(bucket, object string, data *hash.Reader,
+func (a GatewayUnsupported) AnonPutObject(bucket, object string, data *hash.Reader,
 	metadata map[string]string) (ObjectInfo, error) {
-	return ObjectInfo{}, traceError(NotImplemented{})
+	return ObjectInfo{}, errors.Trace(NotImplemented{})
+}
+
+// AnonGetObject downloads object anonymously.
+func (a GatewayUnsupported) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
+	return errors.Trace(NotImplemented{})
+}
+
+// AnonGetObjectInfo returns stat information about an object anonymously.
+func (a GatewayUnsupported) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
+	return objInfo, errors.Trace(NotImplemented{})
+}
+
+// CopyObject copies a blob from source container to destination container.
+func (a GatewayUnsupported) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string,
+	metadata map[string]string) (objInfo ObjectInfo, err error) {
+	return objInfo, errors.Trace(NotImplemented{})
 }

cmd/gateway/azure/gateway-azure-anonymous.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package azure
 
 import (
 	"encoding/xml"
@@ -28,6 +28,9 @@ import (
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/minio/minio/pkg/errors"
+
+	minio "github.com/minio/minio/cmd"
 )
 
 // Copied from github.com/Azure/azure-sdk-for-go/storage/container.go
@@ -112,34 +115,32 @@ func azureAnonRequest(verb, urlStr string, header http.Header) (*http.Response,
 }
 
 // AnonGetBucketInfo - Get bucket metadata from azure anonymously.
-func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
+func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo minio.BucketInfo, err error) {
 	blobURL := a.client.GetContainerReference(bucket).GetBlobReference("").GetURL()
 	url, err := url.Parse(blobURL)
 	if err != nil {
-		return bucketInfo, azureToObjectError(traceError(err))
+		return bucketInfo, azureToObjectError(errors.Trace(err))
 	}
 	url.RawQuery = "restype=container"
-	resp, err := azureAnonRequest(httpHEAD, url.String(), nil)
+	resp, err := azureAnonRequest(http.MethodHead, url.String(), nil)
 	if err != nil {
-		return bucketInfo, azureToObjectError(traceError(err), bucket)
+		return bucketInfo, azureToObjectError(errors.Trace(err), bucket)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return bucketInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket)), bucket)
+		return bucketInfo, azureToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket)), bucket)
 	}
 
 	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
 	if err != nil {
-		return bucketInfo, traceError(err)
+		return bucketInfo, errors.Trace(err)
 	}
 
-	bucketInfo = BucketInfo{
+	return minio.BucketInfo{
 		Name:    bucket,
 		Created: t,
-	}
-
-	return bucketInfo, nil
+	}, nil
 }
 
 // AnonGetObject - SendGET request without authentication.
@@ -153,32 +154,32 @@ func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, l
 	}
 
 	blobURL := a.client.GetContainerReference(bucket).GetBlobReference(object).GetURL()
-	resp, err := azureAnonRequest(httpGET, blobURL, h)
+	resp, err := azureAnonRequest(http.MethodGet, blobURL, h)
 	if err != nil {
-		return azureToObjectError(traceError(err), bucket, object)
+		return azureToObjectError(errors.Trace(err), bucket, object)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
-		return azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
+		return azureToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
 	}
 
 	_, err = io.Copy(writer, resp.Body)
-	return traceError(err)
+	return errors.Trace(err)
 }
 
 // AnonGetObjectInfo - Send HEAD request without authentication and convert the
 // result to ObjectInfo.
-func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
+func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo minio.ObjectInfo, err error) {
 	blobURL := a.client.GetContainerReference(bucket).GetBlobReference(object).GetURL()
-	resp, err := azureAnonRequest(httpHEAD, blobURL, nil)
+	resp, err := azureAnonRequest(http.MethodHead, blobURL, nil)
 	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, object)
+		return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return objInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
+		return objInfo, azureToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
 	}
 
 	var contentLength int64
@@ -186,13 +187,13 @@ func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectI
 	if contentLengthStr != "" {
 		contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
 		if err != nil {
-			return objInfo, azureToObjectError(traceError(errUnexpected), bucket, object)
+			return objInfo, azureToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object)
 		}
 	}
 
 	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
 	if err != nil {
-		return objInfo, traceError(err)
+		return objInfo, errors.Trace(err)
 	}
 
 	objInfo.ModTime = t
@@ -210,7 +211,7 @@ func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectI
 }
 
 // AnonListObjects - Use Azure equivalent ListBlobs.
-func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
+func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) {
 	params := storage.ListBlobsParameters{
 		Prefix:     prefix,
 		Marker:     marker,
@@ -225,13 +226,13 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 	blobURL := a.client.GetContainerReference(bucket).GetBlobReference("").GetURL()
 	url, err := url.Parse(blobURL)
 	if err != nil {
-		return result, azureToObjectError(traceError(err))
+		return result, azureToObjectError(errors.Trace(err))
 	}
 	url.RawQuery = q.Encode()
 
-	resp, err := azureAnonRequest(httpGET, url.String(), nil)
+	resp, err := azureAnonRequest(http.MethodGet, url.String(), nil)
 	if err != nil {
-		return result, azureToObjectError(traceError(err))
+		return result, azureToObjectError(errors.Trace(err))
 	}
 	defer resp.Body.Close()
 
@@ -239,17 +240,17 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return result, azureToObjectError(traceError(err))
+		return result, azureToObjectError(errors.Trace(err))
 	}
 	err = xml.Unmarshal(data, &listResp)
 	if err != nil {
-		return result, azureToObjectError(traceError(err))
+		return result, azureToObjectError(errors.Trace(err))
 	}
 
 	result.IsTruncated = listResp.NextMarker != ""
 	result.NextMarker = listResp.NextMarker
 	for _, object := range listResp.Blobs {
-		result.Objects = append(result.Objects, ObjectInfo{
+		result.Objects = append(result.Objects, minio.ObjectInfo{
 			Bucket:          bucket,
 			Name:            object.Name,
 			ModTime:         time.Time(object.Properties.LastModified),
@@ -264,7 +265,7 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 }
 
 // AnonListObjectsV2 - List objects in V2 mode, anonymously
-func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
+func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) {
 	params := storage.ListBlobsParameters{
 		Prefix:     prefix,
 		Marker:     continuationToken,
@@ -279,13 +280,13 @@ func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, deli
 	blobURL := a.client.GetContainerReference(bucket).GetBlobReference("").GetURL()
 	url, err := url.Parse(blobURL)
 	if err != nil {
-		return result, azureToObjectError(traceError(err))
+		return result, azureToObjectError(errors.Trace(err))
 	}
 	url.RawQuery = q.Encode()
 
 	resp, err := http.Get(url.String())
 	if err != nil {
-		return result, azureToObjectError(traceError(err))
+		return result, azureToObjectError(errors.Trace(err))
 	}
 	defer resp.Body.Close()
 
@@ -293,11 +294,11 @@ func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, deli
 
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return result, azureToObjectError(traceError(err))
+		return result, azureToObjectError(errors.Trace(err))
 	}
 	err = xml.Unmarshal(data, &listResp)
 	if err != nil {
-		return result, azureToObjectError(traceError(err))
+		return result, azureToObjectError(errors.Trace(err))
 	}
 
 	// If NextMarker is not empty, this means response is truncated and NextContinuationToken should be set
@@ -306,12 +307,12 @@ func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, deli
 		result.NextContinuationToken = listResp.NextMarker
 	}
 	for _, object := range listResp.Blobs {
-		result.Objects = append(result.Objects, ObjectInfo{
+		result.Objects = append(result.Objects, minio.ObjectInfo{
 			Bucket:          bucket,
 			Name:            object.Name,
 			ModTime:         time.Time(object.Properties.LastModified),
 			Size:            object.Properties.ContentLength,
-			ETag:            canonicalizeETag(object.Properties.Etag),
+			ETag:            minio.CanonicalizeETag(object.Properties.Etag),
 			ContentType:     object.Properties.ContentType,
 			ContentEncoding: object.Properties.ContentEncoding,
 		})

cmd/gateway/azure/gateway-azure.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package azure
 
 import (
 	"bytes"
@@ -23,23 +23,128 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/cli"
 	"github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
+
+	minio "github.com/minio/minio/cmd"
 )
 
-const globalAzureAPIVersion = "2016-05-31"
-const azureBlockSize = 100 * humanize.MiByte
-const metadataObjectNameTemplate = globalMinioSysTmp + "multipart/v1/%s.%x/azure.json"
+const (
+	globalAzureAPIVersion      = "2016-05-31"
+	azureBlockSize             = 100 * humanize.MiByte
+	azureS3MinPartSize         = 5 * humanize.MiByte
+	metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
+	azureBackend               = "azure"
+)
+
+func init() {
+	const azureGatewayTemplate = `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
+{{if .VisibleFlags}}
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}{{end}}
+ENDPOINT:
+  Azure server endpoint. Default ENDPOINT is https://core.windows.net
+
+ENVIRONMENT VARIABLES:
+  ACCESS:
+     MINIO_ACCESS_KEY: Username or access key of Azure storage.
+     MINIO_SECRET_KEY: Password or secret key of Azure storage.
+
+  BROWSER:
+     MINIO_BROWSER: To disable web browser access, set this value to "off".
+
+  UPDATE:
+     MINIO_UPDATE: To turn off in-place upgrades, set this value to "off".
+
+EXAMPLES:
+  1. Start minio gateway server for Azure Blob Storage backend.
+      $ export MINIO_ACCESS_KEY=azureaccountname
+      $ export MINIO_SECRET_KEY=azureaccountkey
+      $ {{.HelpName}}
+
+  2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
+      $ export MINIO_ACCESS_KEY=azureaccountname
+      $ export MINIO_SECRET_KEY=azureaccountkey
+      $ {{.HelpName}} https://azure.example.com
+
+`
+
+	minio.RegisterGatewayCommand(cli.Command{
+		Name:               azureBackend,
+		Usage:              "Microsoft Azure Blob Storage.",
+		Action:             azureGatewayMain,
+		CustomHelpTemplate: azureGatewayTemplate,
+		HideHelpCommand:    true,
+	})
+}
+
+// Handler for 'minio gateway azure' command line.
+func azureGatewayMain(ctx *cli.Context) {
+	// Validate gateway arguments.
+	host := ctx.Args().First()
+	// Validate gateway arguments.
+	minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
+
+	minio.StartGateway(ctx, &Azure{host})
+}
+
+// Azure implements Gateway.
+type Azure struct {
+	host string
+}
+
+// Name implements Gateway interface.
+func (g *Azure) Name() string {
+	return azureBackend
+}
+
+// NewGatewayLayer initializes azure blob storage client and returns AzureObjects.
+func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.GatewayLayer, error) {
+	var err error
+	var endpoint = storage.DefaultBaseURL
+	var secure = true
+
+	// If user provided some parameters
+	if g.host != "" {
+		endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	c, err := storage.NewClient(creds.AccessKey, creds.SecretKey, endpoint, globalAzureAPIVersion, secure)
+	if err != nil {
+		return &azureObjects{}, err
+	}
+	c.HTTPClient = &http.Client{Transport: minio.NewCustomHTTPTransport()}
+
+	return &azureObjects{
+		client: c.GetBlobService(),
+	}, nil
+}
+
+// Production - Azure gateway is production ready.
+func (g *Azure) Production() bool {
+	return true
+}
 
 // s3MetaToAzureProperties converts metadata meant for S3 PUT/COPY
 // object into Azure data structures - BlobMetadata and
@@ -59,7 +164,7 @@ func s3MetaToAzureProperties(s3Metadata map[string]string) (storage.BlobMetadata
 	storage.BlobProperties, error) {
 	for k := range s3Metadata {
 		if strings.Contains(k, "--") {
-			return storage.BlobMetadata{}, storage.BlobProperties{}, traceError(UnsupportedMetadata{})
+			return storage.BlobMetadata{}, storage.BlobProperties{}, errors.Trace(minio.UnsupportedMetadata{})
 		}
 	}
 
@@ -164,7 +269,7 @@ func azurePropertiesToS3Meta(meta storage.BlobMetadata, props storage.BlobProper
 
 // azureObjects - Implements Object layer for Azure blob storage.
 type azureObjects struct {
-	gatewayUnsupported
+	minio.GatewayUnsupported
 	client storage.BlobStorageClient // Azure sdk client
 }
 
@@ -174,15 +279,15 @@ func azureToObjectError(err error, params ...string) error {
 		return nil
 	}
 
-	e, ok := err.(*Error)
+	e, ok := err.(*errors.Error)
 	if !ok {
-		// Code should be fixed if this function is called without doing traceError()
+		// Code should be fixed if this function is called without doing errors.Trace()
 		// Else handling different situations in this function makes this function complicated.
-		errorIf(err, "Expected type *Error")
+		minio.ErrorIf(err, "Expected type *Error")
 		return err
 	}
 
-	err = e.e
+	err = e.Cause
 	bucket := ""
 	object := ""
 	if len(params) >= 1 {
@@ -201,26 +306,29 @@ func azureToObjectError(err error, params ...string) error {
 
 	switch azureErr.Code {
 	case "ContainerAlreadyExists":
-		err = BucketExists{Bucket: bucket}
+		err = minio.BucketExists{Bucket: bucket}
 	case "InvalidResourceName":
-		err = BucketNameInvalid{Bucket: bucket}
+		err = minio.BucketNameInvalid{Bucket: bucket}
 	case "RequestBodyTooLarge":
-		err = PartTooBig{}
+		err = minio.PartTooBig{}
 	case "InvalidMetadata":
-		err = UnsupportedMetadata{}
+		err = minio.UnsupportedMetadata{}
 	default:
 		switch azureErr.StatusCode {
 		case http.StatusNotFound:
 			if object != "" {
-				err = ObjectNotFound{bucket, object}
+				err = minio.ObjectNotFound{
+					Bucket: bucket,
+					Object: object,
+				}
 			} else {
-				err = BucketNotFound{Bucket: bucket}
+				err = minio.BucketNotFound{Bucket: bucket}
 			}
 		case http.StatusBadRequest:
-			err = BucketNameInvalid{Bucket: bucket}
+			err = minio.BucketNameInvalid{Bucket: bucket}
 		}
 	}
-	e.e = err
+	e.Cause = err
 	return e
 }
 
@@ -242,11 +350,15 @@ func mustGetAzureUploadID() string {
 // checkAzureUploadID - returns error in case of given string is upload ID.
 func checkAzureUploadID(uploadID string) (err error) {
 	if len(uploadID) != 16 {
-		return traceError(MalformedUploadID{uploadID})
+		return errors.Trace(minio.MalformedUploadID{
+			UploadID: uploadID,
+		})
 	}
 
 	if _, err = hex.DecodeString(uploadID); err != nil {
-		return traceError(MalformedUploadID{uploadID})
+		return errors.Trace(minio.MalformedUploadID{
+			UploadID: uploadID,
+		})
 	}
 
 	return nil
@@ -286,32 +398,6 @@ func azureParseBlockID(blockID string) (partID, subPartNumber int, uploadID, md5
 	return
 }
 
-// Inits azure blob storage client and returns AzureObjects.
-func newAzureLayer(host string) (GatewayLayer, error) {
-	var err error
-	var endpoint = storage.DefaultBaseURL
-	var secure = true
-
-	// If user provided some parameters
-	if host != "" {
-		endpoint, secure, err = parseGatewayEndpoint(host)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	creds := serverConfig.GetCredential()
-	c, err := storage.NewClient(creds.AccessKey, creds.SecretKey, endpoint, globalAzureAPIVersion, secure)
-	if err != nil {
-		return &azureObjects{}, err
-	}
-	c.HTTPClient = &http.Client{Transport: newCustomHTTPTransport()}
-
-	return &azureObjects{
-		client: c.GetBlobService(),
-	}, nil
-}
-
 // Shutdown - save any gateway metadata to disk
 // if necessary and reload upon next restart.
 func (a *azureObjects) Shutdown() error {
@@ -319,7 +405,7 @@ func (a *azureObjects) Shutdown() error {
 }
 
 // StorageInfo - Not relevant to Azure backend.
-func (a *azureObjects) StorageInfo() (si StorageInfo) {
+func (a *azureObjects) StorageInfo() (si minio.StorageInfo) {
 	return si
 }
 
@@ -329,17 +415,17 @@ func (a *azureObjects) MakeBucketWithLocation(bucket, location string) error {
 	err := container.Create(&storage.CreateContainerOptions{
 		Access: storage.ContainerAccessTypePrivate,
 	})
-	return azureToObjectError(traceError(err), bucket)
+	return azureToObjectError(errors.Trace(err), bucket)
 }
 
 // GetBucketInfo - Get bucket metadata..
-func (a *azureObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
+func (a *azureObjects) GetBucketInfo(bucket string) (bi minio.BucketInfo, e error) {
 	// Verify if bucket (container-name) is valid.
 	// IsValidBucketName has same restrictions as container names mentioned
 	// in azure documentation, so we will simply use the same function here.
 	// Ref - https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata
-	if !IsValidBucketName(bucket) {
-		return bi, traceError(BucketNameInvalid{Bucket: bucket})
+	if !minio.IsValidBucketName(bucket) {
+		return bi, errors.Trace(minio.BucketNameInvalid{Bucket: bucket})
 	}
 
 	// Azure does not have an equivalent call, hence use
@@ -348,34 +434,34 @@ func (a *azureObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
 		Prefix: bucket,
 	})
 	if err != nil {
-		return bi, azureToObjectError(traceError(err), bucket)
+		return bi, azureToObjectError(errors.Trace(err), bucket)
 	}
 	for _, container := range resp.Containers {
 		if container.Name == bucket {
 			t, e := time.Parse(time.RFC1123, container.Properties.LastModified)
 			if e == nil {
-				return BucketInfo{
+				return minio.BucketInfo{
 					Name:    bucket,
 					Created: t,
 				}, nil
 			} // else continue
 		}
 	}
-	return bi, traceError(BucketNotFound{Bucket: bucket})
+	return bi, errors.Trace(minio.BucketNotFound{Bucket: bucket})
 }
 
 // ListBuckets - Lists all azure containers, uses Azure equivalent ListContainers.
-func (a *azureObjects) ListBuckets() (buckets []BucketInfo, err error) {
+func (a *azureObjects) ListBuckets() (buckets []minio.BucketInfo, err error) {
 	resp, err := a.client.ListContainers(storage.ListContainersParameters{})
 	if err != nil {
-		return nil, azureToObjectError(traceError(err))
+		return nil, azureToObjectError(errors.Trace(err))
 	}
 	for _, container := range resp.Containers {
 		t, e := time.Parse(time.RFC1123, container.Properties.LastModified)
 		if e != nil {
-			return nil, traceError(e)
+			return nil, errors.Trace(e)
 		}
-		buckets = append(buckets, BucketInfo{
+		buckets = append(buckets, minio.BucketInfo{
 			Name:    container.Name,
 			Created: t,
 		})
@@ -386,13 +472,13 @@ func (a *azureObjects) ListBuckets() (buckets []BucketInfo, err error) {
 // DeleteBucket - delete a container on azure, uses Azure equivalent DeleteContainer.
 func (a *azureObjects) DeleteBucket(bucket string) error {
 	container := a.client.GetContainerReference(bucket)
-	return azureToObjectError(traceError(container.Delete(nil)), bucket)
+	return azureToObjectError(errors.Trace(container.Delete(nil)), bucket)
 }
 
 // ListObjects - lists all blobs on azure with in a container filtered by prefix
 // and marker, uses Azure equivalent ListBlobs.
-func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
-	var objects []ObjectInfo
+func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) {
+	var objects []minio.ObjectInfo
 	var prefixes []string
 	container := a.client.GetContainerReference(bucket)
 	for len(objects) == 0 && len(prefixes) == 0 {
@@ -403,19 +489,19 @@ func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, max
 			MaxResults: uint(maxKeys),
 		})
 		if err != nil {
-			return result, azureToObjectError(traceError(err), bucket, prefix)
+			return result, azureToObjectError(errors.Trace(err), bucket, prefix)
 		}
 
 		for _, object := range resp.Blobs {
-			if strings.HasPrefix(object.Name, globalMinioSysTmp) {
+			if strings.HasPrefix(object.Name, minio.GatewayMinioSysTmp) {
 				continue
 			}
-			objects = append(objects, ObjectInfo{
+			objects = append(objects, minio.ObjectInfo{
 				Bucket:          bucket,
 				Name:            object.Name,
 				ModTime:         time.Time(object.Properties.LastModified),
 				Size:            object.Properties.ContentLength,
-				ETag:            toS3ETag(object.Properties.Etag),
+				ETag:            minio.ToS3ETag(object.Properties.Etag),
 				ContentType:     object.Properties.ContentType,
 				ContentEncoding: object.Properties.ContentEncoding,
 			})
@@ -423,7 +509,7 @@ func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, max
 
 		// Remove minio.sys.tmp prefix.
 		for _, prefix := range resp.BlobPrefixes {
-			if prefix != globalMinioSysTmp {
+			if prefix != minio.GatewayMinioSysTmp {
 				prefixes = append(prefixes, prefix)
 			}
 		}
@@ -442,13 +528,13 @@ func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, max
 }
 
 // ListObjectsV2 - list all blobs in Azure bucket filtered by prefix
-func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
+func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) {
 	marker := continuationToken
 	if startAfter != "" {
 		marker = startAfter
 	}
 
-	var resultV1 ListObjectsInfo
+	var resultV1 minio.ListObjectsInfo
 	resultV1, err = a.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		return result, err
@@ -471,7 +557,7 @@ func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken, delimite
 func (a *azureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
 	// startOffset cannot be negative.
 	if startOffset < 0 {
-		return toObjectErr(traceError(errUnexpected), bucket, object)
+		return azureToObjectError(errors.Trace(minio.InvalidRange{}), bucket, object)
 	}
 
 	blobRange := &storage.BlobRange{Start: uint64(startOffset)}
@@ -490,39 +576,37 @@ func (a *azureObjects) GetObject(bucket, object string, startOffset int64, lengt
 		})
 	}
 	if err != nil {
-		return azureToObjectError(traceError(err), bucket, object)
+		return azureToObjectError(errors.Trace(err), bucket, object)
 	}
 	_, err = io.Copy(writer, rc)
 	rc.Close()
-	return traceError(err)
+	return errors.Trace(err)
 }
 
-// GetObjectInfo - reads blob metadata properties and replies back ObjectInfo,
+// GetObjectInfo - reads blob metadata properties and replies back minio.ObjectInfo,
 // uses zure equivalent GetBlobProperties.
-func (a *azureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
+func (a *azureObjects) GetObjectInfo(bucket, object string) (objInfo minio.ObjectInfo, err error) {
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 	err = blob.GetProperties(nil)
 	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, object)
+		return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
 	}
 
-	meta := azurePropertiesToS3Meta(blob.Metadata, blob.Properties)
-	objInfo = ObjectInfo{
+	return minio.ObjectInfo{
 		Bucket:          bucket,
-		UserDefined:     meta,
-		ETag:            toS3ETag(blob.Properties.Etag),
+		UserDefined:     azurePropertiesToS3Meta(blob.Metadata, blob.Properties),
+		ETag:            minio.ToS3ETag(blob.Properties.Etag),
 		ModTime:         time.Time(blob.Properties.LastModified),
 		Name:            object,
 		Size:            blob.Properties.ContentLength,
 		ContentType:     blob.Properties.ContentType,
 		ContentEncoding: blob.Properties.ContentEncoding,
-	}
-	return objInfo, nil
+	}, nil
 }
 
 // PutObject - Create a new blob with the incoming data,
 // uses Azure equivalent CreateBlockBlobFromReader.
-func (a *azureObjects) PutObject(bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
+func (a *azureObjects) PutObject(bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 	blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(metadata)
 	if err != nil {
@@ -530,14 +614,14 @@ func (a *azureObjects) PutObject(bucket, object string, data *hash.Reader, metad
 	}
 	err = blob.CreateBlockBlobFromReader(data, nil)
 	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, object)
+		return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
 	}
 	return a.GetObjectInfo(bucket, object)
 }
 
 // CopyObject - Copies a blob from source container to destination container.
 // Uses Azure equivalent CopyBlob API.
-func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
+func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
 	srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL()
 	destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject)
 	azureMeta, props, err := s3MetaToAzureProperties(metadata)
@@ -547,12 +631,12 @@ func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject s
 	destBlob.Metadata = azureMeta
 	err = destBlob.Copy(srcBlobURL, nil)
 	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), srcBucket, srcObject)
+		return objInfo, azureToObjectError(errors.Trace(err), srcBucket, srcObject)
 	}
 	destBlob.Properties = props
 	err = destBlob.SetProperties(nil)
 	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), srcBucket, srcObject)
+		return objInfo, azureToObjectError(errors.Trace(err), srcBucket, srcObject)
 	}
 	return a.GetObjectInfo(destBucket, destObject)
 }
@@ -563,13 +647,13 @@ func (a *azureObjects) DeleteObject(bucket, object string) error {
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 	err := blob.Delete(nil)
 	if err != nil {
-		return azureToObjectError(traceError(err), bucket, object)
+		return azureToObjectError(errors.Trace(err), bucket, object)
 	}
 	return nil
 }
 
 // ListMultipartUploads - It's decided not to support List Multipart Uploads, hence returning empty result.
-func (a *azureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
+func (a *azureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result minio.ListMultipartsInfo, err error) {
 	// It's decided not to support List Multipart Uploads, hence returning empty result.
 	return result, nil
 }
@@ -587,10 +671,15 @@ func (a *azureObjects) checkUploadIDExists(bucketName, objectName, uploadID stri
 	blob := a.client.GetContainerReference(bucketName).GetBlobReference(
 		getAzureMetadataObjectName(objectName, uploadID))
 	err = blob.GetMetadata(nil)
-	err = azureToObjectError(traceError(err), bucketName, objectName)
-	oerr := ObjectNotFound{bucketName, objectName}
-	if errorCause(err) == oerr {
-		err = traceError(InvalidUploadID{})
+	err = azureToObjectError(errors.Trace(err), bucketName, objectName)
+	oerr := minio.ObjectNotFound{
+		Bucket: bucketName,
+		Object: objectName,
+	}
+	if errors.Cause(err) == oerr {
+		err = errors.Trace(minio.InvalidUploadID{
+			UploadID: uploadID,
+		})
 	}
 	return err
 }
@@ -599,26 +688,26 @@ func (a *azureObjects) checkUploadIDExists(bucketName, objectName, uploadID stri
 func (a *azureObjects) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) {
 	uploadID = mustGetAzureUploadID()
 	if err = a.checkUploadIDExists(bucket, object, uploadID); err == nil {
-		return "", traceError(errors.New("Upload ID name collision"))
+		return "", errors.Trace(fmt.Errorf("Upload ID name collision"))
 	}
 	metadataObject := getAzureMetadataObjectName(object, uploadID)
 
 	var jsonData []byte
 	if jsonData, err = json.Marshal(azureMultipartMetadata{Name: object, Metadata: metadata}); err != nil {
-		return "", traceError(err)
+		return "", errors.Trace(err)
 	}
 
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
 	err = blob.CreateBlockBlobFromReader(bytes.NewBuffer(jsonData), nil)
 	if err != nil {
-		return "", azureToObjectError(traceError(err), bucket, metadataObject)
+		return "", azureToObjectError(errors.Trace(err), bucket, metadataObject)
 	}
 
 	return uploadID, nil
 }
 
 // PutObjectPart - Use Azure equivalent PutBlockWithLength.
-func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
+func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (info minio.PartInfo, err error) {
 	if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
 		return info, err
 	}
@@ -629,7 +718,7 @@ func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int
 
 	etag := data.MD5HexString()
 	if etag == "" {
-		etag = genETag()
+		etag = minio.GenETag()
 	}
 
 	subPartSize, subPartNumber := int64(azureBlockSize), 1
@@ -647,25 +736,99 @@ func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 		err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil)
 		if err != nil {
-			return info, azureToObjectError(traceError(err), bucket, object)
+			return info, azureToObjectError(errors.Trace(err), bucket, object)
 		}
 		subPartNumber++
 	}
 
 	info.PartNumber = partID
 	info.ETag = etag
-	info.LastModified = UTCNow()
+	info.LastModified = minio.UTCNow()
 	info.Size = data.Size()
 	return info, nil
 }
 
 // ListObjectParts - Use Azure equivalent GetBlockList.
-func (a *azureObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
+func (a *azureObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result minio.ListPartsInfo, err error) {
 	if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
 		return result, err
 	}
 
-	// It's decided not to support List Object Parts, hence returning empty result.
+	result.Bucket = bucket
+	result.Object = object
+	result.UploadID = uploadID
+	result.MaxParts = maxParts
+
+	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
+	resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
+	azureErr, ok := err.(storage.AzureStorageServiceError)
+	if ok && azureErr.StatusCode == http.StatusNotFound {
+		// If no parts are uploaded yet then we return empty list.
+		return result, nil
+	}
+	if err != nil {
+		return result, azureToObjectError(errors.Trace(err), bucket, object)
+	}
+	// Build a sorted list of parts and return the requested entries.
+	partsMap := make(map[int]minio.PartInfo)
+	for _, block := range resp.UncommittedBlocks {
+		var partNumber int
+		var parsedUploadID string
+		var md5Hex string
+		if partNumber, _, parsedUploadID, md5Hex, err = azureParseBlockID(block.Name); err != nil {
+			return result, azureToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object)
+		}
+		if parsedUploadID != uploadID {
+			continue
+		}
+		part, ok := partsMap[partNumber]
+		if !ok {
+			partsMap[partNumber] = minio.PartInfo{
+				PartNumber: partNumber,
+				Size:       block.Size,
+				ETag:       md5Hex,
+			}
+			continue
+		}
+		if part.ETag != md5Hex {
+			// If two parts of same partNumber were uploaded with different contents
+			// return error as we won't be able to decide which the latest part is.
+			return result, azureToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object)
+		}
+		part.Size += block.Size
+		partsMap[partNumber] = part
+	}
+	var parts []minio.PartInfo
+	for _, part := range partsMap {
+		parts = append(parts, part)
+	}
+	sort.SliceStable(parts, func(i int, j int) bool {
+		return parts[i].PartNumber < parts[j].PartNumber
+	})
+	partsCount := 0
+	i := 0
+	if partNumberMarker != 0 {
+		// If the marker was set, skip the entries till the marker.
+		for _, part := range parts {
+			i++
+			if part.PartNumber == partNumberMarker {
+				break
+			}
+		}
+	}
+	for partsCount < maxParts && i < len(parts) {
+		result.Parts = append(result.Parts, parts[i])
+		i++
+		partsCount++
+	}
+
+	if i < len(parts) {
+		result.IsTruncated = true
+		if partsCount != 0 {
+			result.NextPartNumberMarker = result.Parts[partsCount-1].PartNumber
+		}
+	}
+	result.PartNumberMarker = partNumberMarker
 	return result, nil
 }
 
@@ -683,7 +846,7 @@ func (a *azureObjects) AbortMultipartUpload(bucket, object, uploadID string) (er
 }
 
 // CompleteMultipartUpload - Use Azure equivalent PutBlockList.
-func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (objInfo ObjectInfo, err error) {
+func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []minio.CompletePart) (objInfo minio.ObjectInfo, err error) {
 	metadataObject := getAzureMetadataObjectName(object, uploadID)
 	if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
 		return objInfo, err
@@ -696,12 +859,12 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
 	var metadataReader io.Reader
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
 	if metadataReader, err = blob.Get(nil); err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, metadataObject)
+		return objInfo, azureToObjectError(errors.Trace(err), bucket, metadataObject)
 	}
 
 	var metadata azureMultipartMetadata
 	if err = json.NewDecoder(metadataReader).Decode(&metadata); err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, metadataObject)
+		return objInfo, azureToObjectError(errors.Trace(err), bucket, metadataObject)
 	}
 
 	defer func() {
@@ -711,13 +874,13 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
 
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
 		derr := blob.Delete(nil)
-		errorIf(derr, "unable to remove meta data object for upload ID %s", uploadID)
+		minio.ErrorIf(derr, "unable to remove meta data object for upload ID %s", uploadID)
 	}()
 
 	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 	resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
 	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, object)
+		return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
 	}
 
 	getBlocks := func(partNumber int, etag string) (blocks []storage.Block, size int64, err error) {
@@ -740,7 +903,7 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
 		}
 
 		if len(blocks) == 0 {
-			return nil, 0, InvalidPart{}
+			return nil, 0, minio.InvalidPart{}
 		}
 
 		return blocks, size, nil
@@ -753,7 +916,7 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
 		var size int64
 		blocks, size, err = getBlocks(part.PartNumber, part.ETag)
 		if err != nil {
-			return objInfo, traceError(err)
+			return objInfo, errors.Trace(err)
 		}
 
 		allBlocks = append(allBlocks, blocks...)
@@ -762,8 +925,8 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
 
 	// Error out if parts except last part sizing < 5MiB.
 	for i, size := range partSizes[:len(partSizes)-1] {
-		if size < globalMinPartSize {
-			return objInfo, traceError(PartTooSmall{
+		if size < azureS3MinPartSize {
+			return objInfo, errors.Trace(minio.PartTooSmall{
 				PartNumber: uploadedParts[i].PartNumber,
 				PartSize:   size,
 				PartETag:   uploadedParts[i].ETag,
@@ -773,7 +936,7 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
 
 	err = objBlob.PutBlockList(allBlocks, nil)
 	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, object)
+		return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
 	}
 	if len(metadata.Metadata) > 0 {
 		objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(metadata.Metadata)
@@ -782,11 +945,11 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
 		}
 		err = objBlob.SetProperties(nil)
 		if err != nil {
-			return objInfo, azureToObjectError(traceError(err), bucket, object)
+			return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
 		}
 		err = objBlob.SetMetadata(nil)
 		if err != nil {
-			return objInfo, azureToObjectError(traceError(err), bucket, object)
+			return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
 		}
 	}
 	return a.GetObjectInfo(bucket, object)
@@ -799,23 +962,23 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
 // As the common denominator for minio and azure is readonly and none, we support
 // these two policies at the bucket level.
 func (a *azureObjects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
-	var policies []BucketAccessPolicy
+	var policies []minio.BucketAccessPolicy
 
 	for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
-		policies = append(policies, BucketAccessPolicy{
+		policies = append(policies, minio.BucketAccessPolicy{
 			Prefix: prefix,
 			Policy: policy,
 		})
 	}
 	prefix := bucket + "/*" // For all objects inside the bucket.
 	if len(policies) != 1 {
-		return traceError(NotImplemented{})
+		return errors.Trace(minio.NotImplemented{})
 	}
 	if policies[0].Prefix != prefix {
-		return traceError(NotImplemented{})
+		return errors.Trace(minio.NotImplemented{})
 	}
 	if policies[0].Policy != policy.BucketPolicyReadOnly {
-		return traceError(NotImplemented{})
+		return errors.Trace(minio.NotImplemented{})
 	}
 	perm := storage.ContainerPermissions{
 		AccessType:     storage.ContainerAccessTypeContainer,
@@ -823,7 +986,7 @@ func (a *azureObjects) SetBucketPolicies(bucket string, policyInfo policy.Bucket
 	}
 	container := a.client.GetContainerReference(bucket)
 	err := container.SetPermissions(perm, nil)
-	return azureToObjectError(traceError(err), bucket)
+	return azureToObjectError(errors.Trace(err), bucket)
 }
 
 // GetBucketPolicies - Get the container ACL and convert it to canonical []bucketAccessPolicy
@@ -832,15 +995,15 @@ func (a *azureObjects) GetBucketPolicies(bucket string) (policy.BucketAccessPoli
 	container := a.client.GetContainerReference(bucket)
 	perm, err := container.GetPermissions(nil)
 	if err != nil {
-		return policy.BucketAccessPolicy{}, azureToObjectError(traceError(err), bucket)
+		return policy.BucketAccessPolicy{}, azureToObjectError(errors.Trace(err), bucket)
 	}
 	switch perm.AccessType {
 	case storage.ContainerAccessTypePrivate:
-		return policy.BucketAccessPolicy{}, traceError(PolicyNotFound{Bucket: bucket})
+		return policy.BucketAccessPolicy{}, errors.Trace(minio.PolicyNotFound{Bucket: bucket})
 	case storage.ContainerAccessTypeContainer:
 		policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
 	default:
-		return policy.BucketAccessPolicy{}, azureToObjectError(traceError(NotImplemented{}))
+		return policy.BucketAccessPolicy{}, azureToObjectError(errors.Trace(minio.NotImplemented{}))
 	}
 	return policyInfo, nil
 }
@@ -853,5 +1016,5 @@ func (a *azureObjects) DeleteBucketPolicies(bucket string) error {
 	}
 	container := a.client.GetContainerReference(bucket)
 	err := container.SetPermissions(perm, nil)
-	return azureToObjectError(traceError(err))
+	return azureToObjectError(errors.Trace(err))
 }

cmd/gateway/azure/gateway-azure_test.go

@@ -14,15 +14,18 @@
  * limitations under the License.
  */
 
-package cmd
+package azure
 
 import (
+	"fmt"
 	"net/http"
 	"net/url"
 	"reflect"
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
+	minio "github.com/minio/minio/cmd"
+	"github.com/minio/minio/pkg/errors"
 )
 
 // Test canonical metadata.
@@ -30,6 +33,10 @@ func TestS3MetaToAzureProperties(t *testing.T) {
 	headers := map[string]string{
 		"accept-encoding":          "gzip",
 		"content-encoding":         "gzip",
+		"cache-control":            "age: 3600",
+		"content-disposition":      "dummy",
+		"content-length":           "10",
+		"content-type":             "application/javascript",
 		"X-Amz-Meta-Hdr":           "value",
 		"X-Amz-Meta-X_test_key":    "value",
 		"X-Amz-Meta-X__test__key":  "value",
@@ -60,8 +67,8 @@ func TestS3MetaToAzureProperties(t *testing.T) {
 		"invalid--meta": "value",
 	}
 	_, _, err = s3MetaToAzureProperties(headers)
-	if err = errorCause(err); err != nil {
-		if _, ok := err.(UnsupportedMetadata); !ok {
+	if err = errors.Cause(err); err != nil {
+		if _, ok := err.(minio.UnsupportedMetadata); !ok {
 			t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err)
 		}
 	}
@@ -100,8 +107,21 @@ func TestAzurePropertiesToS3Meta(t *testing.T) {
 		"X-Amz-Meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
 		"X-Amz-Meta-X-Amz-Matdesc": "{}",
 		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
+		"Cache-Control":            "max-age: 3600",
+		"Content-Disposition":      "dummy",
+		"Content-Encoding":         "gzip",
+		"Content-Length":           "10",
+		"Content-MD5":              "base64-md5",
+		"Content-Type":             "application/javascript",
 	}
-	actualMeta := azurePropertiesToS3Meta(metadata, storage.BlobProperties{})
+	actualMeta := azurePropertiesToS3Meta(metadata, storage.BlobProperties{
+		CacheControl:       "max-age: 3600",
+		ContentDisposition: "dummy",
+		ContentEncoding:    "gzip",
+		ContentLength:      10,
+		ContentMD5:         "base64-md5",
+		ContentType:        "application/javascript",
+	})
 	if !reflect.DeepEqual(actualMeta, expectedMeta) {
 		t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, actualMeta)
 	}
@@ -118,43 +138,57 @@ func TestAzureToObjectError(t *testing.T) {
 			nil, nil, "", "",
 		},
 		{
-			traceError(errUnexpected), errUnexpected, "", "",
+			errors.Trace(fmt.Errorf("Non azure error")),
+			fmt.Errorf("Non azure error"), "", "",
 		},
 		{
-			traceError(errUnexpected), traceError(errUnexpected), "", "",
-		},
-		{
-			traceError(storage.AzureStorageServiceError{
+			storage.AzureStorageServiceError{
 				Code: "ContainerAlreadyExists",
-			}), BucketExists{Bucket: "bucket"}, "bucket", "",
+			}, storage.AzureStorageServiceError{
+				Code: "ContainerAlreadyExists",
+			}, "bucket", "",
 		},
 		{
-			traceError(storage.AzureStorageServiceError{
+			errors.Trace(storage.AzureStorageServiceError{
+				Code: "ContainerAlreadyExists",
+			}), minio.BucketExists{Bucket: "bucket"}, "bucket", "",
+		},
+		{
+			errors.Trace(storage.AzureStorageServiceError{
 				Code: "InvalidResourceName",
-			}), BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
+			}), minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
 		},
 		{
-			traceError(storage.AzureStorageServiceError{
+			errors.Trace(storage.AzureStorageServiceError{
+				Code: "RequestBodyTooLarge",
+			}), minio.PartTooBig{}, "", "",
+		},
+		{
+			errors.Trace(storage.AzureStorageServiceError{
+				Code: "InvalidMetadata",
+			}), minio.UnsupportedMetadata{}, "", "",
+		},
+		{
+			errors.Trace(storage.AzureStorageServiceError{
 				StatusCode: http.StatusNotFound,
-			}), ObjectNotFound{
+			}), minio.ObjectNotFound{
 				Bucket: "bucket",
 				Object: "object",
 			}, "bucket", "object",
 		},
 		{
-			traceError(storage.AzureStorageServiceError{
+			errors.Trace(storage.AzureStorageServiceError{
 				StatusCode: http.StatusNotFound,
-			}), BucketNotFound{Bucket: "bucket"}, "bucket", "",
+			}), minio.BucketNotFound{Bucket: "bucket"}, "bucket", "",
 		},
 		{
-			traceError(storage.AzureStorageServiceError{
+			errors.Trace(storage.AzureStorageServiceError{
 				StatusCode: http.StatusBadRequest,
-			}), BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
+			}), minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
 		},
 	}
 	for i, testCase := range testCases {
-		err := azureToObjectError(testCase.actualErr, testCase.bucket, testCase.object)
-		if err != nil {
+		if err := azureToObjectError(testCase.actualErr, testCase.bucket, testCase.object); err != nil {
 			if err.Error() != testCase.expectedErr.Error() {
 				t.Errorf("Test %d: Expected error %s, got %s", i+1, testCase.expectedErr, err)
 			}
@@ -190,32 +224,41 @@ func TestAzureParseBlockID(t *testing.T) {
 		subPartNumber int
 		uploadID      string
 		md5           string
+		success       bool
 	}{
-		{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ==", 1, 7, "f328c35cad938137", "d41d8cd98f00b204e9800998ecf8427e"},
-		{"MDAwMDIuMTkuYWJjZGMzNWNhZDkzODEzNy5hN2ZiNmI3YjM2ZWU0ZWQ2NmI1NTQ2ZmFjNDY5MDI3Mw==", 2, 19, "abcdc35cad938137", "a7fb6b7b36ee4ed66b5546fac4690273"},
+		// Invalid base64.
+		{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ=", 0, 0, "", "", false},
+		// Invalid number of tokens.
+		{"MDAwMDEuQUEuZjMyOGMzNWNhZDkzODEzNwo=", 0, 0, "", "", false},
+		// Invalid encoded part ID.
+		{"MDAwMGEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQo=", 0, 0, "", "", false},
+		// Invalid sub part ID.
+		{"MDAwMDEuQUEuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQo=", 0, 0, "", "", false},
+		{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ==", 1, 7, "f328c35cad938137", "d41d8cd98f00b204e9800998ecf8427e", true},
+		{"MDAwMDIuMTkuYWJjZGMzNWNhZDkzODEzNy5hN2ZiNmI3YjM2ZWU0ZWQ2NmI1NTQ2ZmFjNDY5MDI3Mw==", 2, 19, "abcdc35cad938137", "a7fb6b7b36ee4ed66b5546fac4690273", true},
 	}
-	for _, test := range testCases {
+	for i, test := range testCases {
 		partID, subPartNumber, uploadID, md5, err := azureParseBlockID(test.blockID)
-		if err != nil {
-			t.Fatal(err)
+		if err != nil && test.success {
+			t.Errorf("Test %d: Expected success but failed %s", i+1, err)
 		}
-		if partID != test.partID {
-			t.Fatalf("%d not equal to %d", partID, test.partID)
+		if err == nil && !test.success {
+			t.Errorf("Test %d: Expected to fail but succeeeded insteadl", i+1)
 		}
-		if subPartNumber != test.subPartNumber {
-			t.Fatalf("%d not equal to %d", subPartNumber, test.subPartNumber)
+		if err == nil {
+			if partID != test.partID {
+				t.Errorf("Test %d: %d not equal to %d", i+1, partID, test.partID)
+			}
+			if subPartNumber != test.subPartNumber {
+				t.Errorf("Test %d: %d not equal to %d", i+1, subPartNumber, test.subPartNumber)
+			}
+			if uploadID != test.uploadID {
+				t.Errorf("Test %d: %s not equal to %s", i+1, uploadID, test.uploadID)
+			}
+			if md5 != test.md5 {
+				t.Errorf("Test %d: %s not equal to %s", i+1, md5, test.md5)
+			}
 		}
-		if uploadID != test.uploadID {
-			t.Fatalf("%s not equal to %s", uploadID, test.uploadID)
-		}
-		if md5 != test.md5 {
-			t.Fatalf("%s not equal to %s", md5, test.md5)
-		}
-	}
-
-	_, _, _, _, err := azureParseBlockID("junk")
-	if err == nil {
-		t.Fatal("Expected azureParseBlockID() to return error")
 	}
 }
 
@@ -279,32 +322,27 @@ func TestAnonErrToObjectErr(t *testing.T) {
 		{"ObjectNotFound",
 			http.StatusNotFound,
 			[]string{"testBucket", "testObject"},
-			ObjectNotFound{Bucket: "testBucket", Object: "testObject"},
+			minio.ObjectNotFound{Bucket: "testBucket", Object: "testObject"},
 		},
 		{"BucketNotFound",
 			http.StatusNotFound,
 			[]string{"testBucket", ""},
-			BucketNotFound{Bucket: "testBucket"},
+			minio.BucketNotFound{Bucket: "testBucket"},
 		},
 		{"ObjectNameInvalid",
 			http.StatusBadRequest,
 			[]string{"testBucket", "testObject"},
-			ObjectNameInvalid{Bucket: "testBucket", Object: "testObject"},
+			minio.ObjectNameInvalid{Bucket: "testBucket", Object: "testObject"},
 		},
 		{"BucketNameInvalid",
 			http.StatusBadRequest,
 			[]string{"testBucket", ""},
-			BucketNameInvalid{Bucket: "testBucket"},
-		},
-		{"UnexpectedError",
-			http.StatusBadGateway,
-			[]string{"testBucket", "testObject"},
-			errUnexpected,
+			minio.BucketNameInvalid{Bucket: "testBucket"},
 		},
 	}
 	for _, test := range testCases {
 		t.Run(test.name, func(t *testing.T) {
-			if err := anonErrToObjectErr(test.statusCode, test.params...); !reflect.DeepEqual(err, test.wantErr) {
+			if err := minio.AnonErrToObjectErr(test.statusCode, test.params...); !reflect.DeepEqual(err, test.wantErr) {
 				t.Errorf("anonErrToObjectErr() error = %v, wantErr %v", err, test.wantErr)
 			}
 		})

cmd/gateway/b2/gateway-b2-anonymous.go

@@ -14,10 +14,9 @@
  * limitations under the License.
  */
 
-package cmd
+package b2
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -25,6 +24,10 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/minio/minio/pkg/errors"
+
+	minio "github.com/minio/minio/cmd"
 )
 
 // mkRange converts offset, size into Range header equivalent.
@@ -33,9 +36,9 @@ func mkRange(offset, size int64) string {
 		return ""
 	}
 	if size == 0 {
-		return fmt.Sprintf("%s%d-", byteRangePrefix, offset)
+		return fmt.Sprintf("bytes=%d-", offset)
 	}
-	return fmt.Sprintf("%s%d-%d", byteRangePrefix, offset, offset+size-1)
+	return fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)
 }
 
 // AnonGetObject - performs a plain http GET request on a public resource,
@@ -44,7 +47,7 @@ func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int6
 	uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
 	req, err := http.NewRequest("GET", uri, nil)
 	if err != nil {
-		return b2ToObjectError(traceError(err), bucket, object)
+		return b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 	rng := mkRange(startOffset, length)
 	if rng != "" {
@@ -52,14 +55,14 @@ func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int6
 	}
 	resp, err := l.anonClient.Do(req)
 	if err != nil {
-		return b2ToObjectError(traceError(err), bucket, object)
+		return b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return b2ToObjectError(traceError(errors.New(resp.Status)), bucket, object)
+		return b2ToObjectError(errors.Trace(fmt.Errorf(resp.Status)), bucket, object)
 	}
 	_, err = io.Copy(writer, resp.Body)
-	return b2ToObjectError(traceError(err), bucket, object)
+	return b2ToObjectError(errors.Trace(err), bucket, object)
 }
 
 // Converts http Header into ObjectInfo. This function looks for all the
@@ -70,16 +73,16 @@ func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int6
 // X-Bz-Info-<header>:<value> is converted to <header>:<value>
 // Content-Type is converted to ContentType.
 // X-Bz-Content-Sha1 is converted to ETag.
-func headerToObjectInfo(bucket, object string, header http.Header) (objInfo ObjectInfo, err error) {
+func headerToObjectInfo(bucket, object string, header http.Header) (objInfo minio.ObjectInfo, err error) {
 	clen, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
 	if err != nil {
-		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 
 	// Converting upload timestamp in milliseconds to a time.Time value for ObjectInfo.ModTime.
 	timeStamp, err := strconv.ParseInt(header.Get("X-Bz-Upload-Timestamp"), 10, 64)
 	if err != nil {
-		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 
 	// Populate user metadata by looking for all the X-Bz-Info-<name>
@@ -91,18 +94,18 @@ func headerToObjectInfo(bucket, object string, header http.Header) (objInfo Obje
 			var name string
 			name, err = url.QueryUnescape(strings.TrimPrefix(key, "X-Bz-Info-"))
 			if err != nil {
-				return objInfo, b2ToObjectError(traceError(err), bucket, object)
+				return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 			}
 			var val string
 			val, err = url.QueryUnescape(header.Get(key))
 			if err != nil {
-				return objInfo, b2ToObjectError(traceError(err), bucket, object)
+				return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 			}
 			userMetadata[name] = val
 		}
 	}
 
-	return ObjectInfo{
+	return minio.ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
 		ContentType: header.Get("Content-Type"),
@@ -115,19 +118,19 @@ func headerToObjectInfo(bucket, object string, header http.Header) (objInfo Obje
 
 // AnonGetObjectInfo - performs a plain http HEAD request on a public resource,
 // fails if the resource is not public.
-func (l *b2Objects) AnonGetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
+func (l *b2Objects) AnonGetObjectInfo(bucket string, object string) (objInfo minio.ObjectInfo, err error) {
 	uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
 	req, err := http.NewRequest("HEAD", uri, nil)
 	if err != nil {
-		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 	resp, err := l.anonClient.Do(req)
 	if err != nil {
-		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return objInfo, b2ToObjectError(traceError(errors.New(resp.Status)), bucket, object)
+		return objInfo, b2ToObjectError(errors.Trace(fmt.Errorf(resp.Status)), bucket, object)
 	}
 	return headerToObjectInfo(bucket, object, resp.Header)
 }

cmd/gateway/b2/gateway-b2.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package b2
 
 import (
 	"context"
@@ -23,39 +23,84 @@ import (
 	"hash"
 	"io"
 	"io/ioutil"
+
 	"net/http"
 	"strings"
 	"sync"
 	"time"
 
 	b2 "github.com/minio/blazer/base"
+	"github.com/minio/cli"
 	"github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/errors"
 	h2 "github.com/minio/minio/pkg/hash"
+
+	minio "github.com/minio/minio/cmd"
 )
 
 // Supported bucket types by B2 backend.
 const (
 	bucketTypePrivate  = "allPrivate"
 	bucketTypeReadOnly = "allPublic"
+	b2Backend          = "b2"
 )
 
-// b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers.
-type b2Objects struct {
-	gatewayUnsupported
-	mu         sync.Mutex
-	creds      credential
-	b2Client   *b2.B2
-	anonClient *http.Client
-	ctx        context.Context
+func init() {
+	const b2GatewayTemplate = `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}}
+{{if .VisibleFlags}}
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}{{end}}
+ENVIRONMENT VARIABLES:
+  ACCESS:
+     MINIO_ACCESS_KEY: B2 account id.
+     MINIO_SECRET_KEY: B2 application key.
+
+  BROWSER:
+     MINIO_BROWSER: To disable web browser access, set this value to "off".
+
+  UPDATE:
+     MINIO_UPDATE: To turn off in-place upgrades, set this value to "off".
+
+EXAMPLES:
+  1. Start minio gateway server for B2 backend.
+      $ export MINIO_ACCESS_KEY=accountID
+      $ export MINIO_SECRET_KEY=applicationKey
+      $ {{.HelpName}}
+
+`
+	minio.RegisterGatewayCommand(cli.Command{
+		Name:               b2Backend,
+		Usage:              "Backblaze B2.",
+		Action:             b2GatewayMain,
+		CustomHelpTemplate: b2GatewayTemplate,
+		HideHelpCommand:    true,
+	})
 }
 
-// newB2Gateway returns b2 gateway layer, implements GatewayLayer interface to
-// talk to B2 remote backend.
-func newB2Gateway() (GatewayLayer, error) {
-	ctx := context.Background()
-	creds := serverConfig.GetCredential()
+// Handler for 'minio gateway b2' command line.
+func b2GatewayMain(ctx *cli.Context) {
+	minio.StartGateway(ctx, &B2{})
+}
 
-	client, err := b2.AuthorizeAccount(ctx, creds.AccessKey, creds.SecretKey, b2.Transport(newCustomHTTPTransport()))
+// B2 implements Minio Gateway
+type B2 struct{}
+
+// Name implements Gateway interface.
+func (g *B2) Name() string {
+	return b2Backend
+}
+
+// NewGatewayLayer returns b2 gateway layer, implements GatewayLayer interface to
+// talk to B2 remote backend.
+func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.GatewayLayer, error) {
+	ctx := context.Background()
+	client, err := b2.AuthorizeAccount(ctx, creds.AccessKey, creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport()))
 	if err != nil {
 		return nil, err
 	}
@@ -64,27 +109,43 @@ func newB2Gateway() (GatewayLayer, error) {
 		creds:    creds,
 		b2Client: client,
 		anonClient: &http.Client{
-			Transport: newCustomHTTPTransport(),
+			Transport: minio.NewCustomHTTPTransport(),
 		},
 		ctx: ctx,
 	}, nil
 }
 
+// Production - Ready for production use?
+func (g *B2) Production() bool {
+	// Not ready for production use just yet.
+	return false
+}
+
+// b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers.
+type b2Objects struct {
+	minio.GatewayUnsupported
+	mu         sync.Mutex
+	creds      auth.Credentials
+	b2Client   *b2.B2
+	anonClient *http.Client
+	ctx        context.Context
+}
+
 // Convert B2 errors to minio object layer errors.
 func b2ToObjectError(err error, params ...string) error {
 	if err == nil {
 		return nil
 	}
 
-	e, ok := err.(*Error)
+	e, ok := err.(*errors.Error)
 	if !ok {
-		// Code should be fixed if this function is called without doing traceError()
+		// Code should be fixed if this function is called without doing errors.Trace()
 		// Else handling different situations in this function makes this function complicated.
-		errorIf(err, "Expected type *Error")
+		minio.ErrorIf(err, "Expected type *Error")
 		return err
 	}
 
-	err = e.e
+	err = e.Cause
 	bucket := ""
 	object := ""
 	uploadID := ""
@@ -111,27 +172,33 @@ func b2ToObjectError(err error, params ...string) error {
 
 	switch code {
 	case "duplicate_bucket_name":
-		err = BucketAlreadyOwnedByYou{Bucket: bucket}
+		err = minio.BucketAlreadyOwnedByYou{Bucket: bucket}
 	case "bad_request":
 		if object != "" {
-			err = ObjectNameInvalid{bucket, object}
+			err = minio.ObjectNameInvalid{
+				Bucket: bucket,
+				Object: object,
+			}
 		} else if bucket != "" {
-			err = BucketNotFound{Bucket: bucket}
+			err = minio.BucketNotFound{Bucket: bucket}
 		}
 	case "bad_bucket_id":
-		err = BucketNotFound{Bucket: bucket}
+		err = minio.BucketNotFound{Bucket: bucket}
 	case "file_not_present", "not_found":
-		err = ObjectNotFound{bucket, object}
+		err = minio.ObjectNotFound{
+			Bucket: bucket,
+			Object: object,
+		}
 	case "cannot_delete_non_empty_bucket":
-		err = BucketNotEmpty{bucket, ""}
+		err = minio.BucketNotEmpty{Bucket: bucket}
 	}
 
 	// Special interpretation like this is required for Multipart sessions.
 	if strings.Contains(msg, "No active upload for") && uploadID != "" {
-		err = InvalidUploadID{uploadID}
+		err = minio.InvalidUploadID{UploadID: uploadID}
 	}
 
-	e.e = err
+	e.Cause = err
 	return e
 }
 
@@ -143,7 +210,7 @@ func (l *b2Objects) Shutdown() error {
 }
 
 // StorageInfo is not relevant to B2 backend.
-func (l *b2Objects) StorageInfo() (si StorageInfo) {
+func (l *b2Objects) StorageInfo() (si minio.StorageInfo) {
 	return si
 }
 
@@ -153,11 +220,11 @@ func (l *b2Objects) MakeBucketWithLocation(bucket, location string) error {
 
 	// All buckets are set to private by default.
 	_, err := l.b2Client.CreateBucket(l.ctx, bucket, bucketTypePrivate, nil, nil)
-	return b2ToObjectError(traceError(err), bucket)
+	return b2ToObjectError(errors.Trace(err), bucket)
 }
 
 func (l *b2Objects) reAuthorizeAccount() error {
-	client, err := b2.AuthorizeAccount(l.ctx, l.creds.AccessKey, l.creds.SecretKey, b2.Transport(newCustomHTTPTransport()))
+	client, err := b2.AuthorizeAccount(l.ctx, l.creds.AccessKey, l.creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport()))
 	if err != nil {
 		return err
 	}
@@ -194,36 +261,36 @@ func (l *b2Objects) listBuckets(err error) ([]*b2.Bucket, error) {
 func (l *b2Objects) Bucket(bucket string) (*b2.Bucket, error) {
 	bktList, err := l.listBuckets(nil)
 	if err != nil {
-		return nil, b2ToObjectError(traceError(err), bucket)
+		return nil, b2ToObjectError(errors.Trace(err), bucket)
 	}
 	for _, bkt := range bktList {
 		if bkt.Name == bucket {
 			return bkt, nil
 		}
 	}
-	return nil, traceError(BucketNotFound{Bucket: bucket})
+	return nil, errors.Trace(minio.BucketNotFound{Bucket: bucket})
 }
 
 // GetBucketInfo gets bucket metadata..
-func (l *b2Objects) GetBucketInfo(bucket string) (bi BucketInfo, err error) {
+func (l *b2Objects) GetBucketInfo(bucket string) (bi minio.BucketInfo, err error) {
 	if _, err = l.Bucket(bucket); err != nil {
 		return bi, err
 	}
-	return BucketInfo{
+	return minio.BucketInfo{
 		Name:    bucket,
 		Created: time.Unix(0, 0),
 	}, nil
 }
 
 // ListBuckets lists all B2 buckets
-func (l *b2Objects) ListBuckets() ([]BucketInfo, error) {
+func (l *b2Objects) ListBuckets() ([]minio.BucketInfo, error) {
 	bktList, err := l.listBuckets(nil)
 	if err != nil {
 		return nil, err
 	}
-	var bktInfo []BucketInfo
+	var bktInfo []minio.BucketInfo
 	for _, bkt := range bktList {
-		bktInfo = append(bktInfo, BucketInfo{
+		bktInfo = append(bktInfo, minio.BucketInfo{
 			Name:    bkt.Name,
 			Created: time.Unix(0, 0),
 		})
@@ -238,19 +305,18 @@ func (l *b2Objects) DeleteBucket(bucket string) error {
 		return err
 	}
 	err = bkt.DeleteBucket(l.ctx)
-	return b2ToObjectError(traceError(err), bucket)
+	return b2ToObjectError(errors.Trace(err), bucket)
 }
 
 // ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time.
-func (l *b2Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
+func (l *b2Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
 		return loi, err
 	}
-	loi = ListObjectsInfo{}
 	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
 	if lerr != nil {
-		return loi, b2ToObjectError(traceError(lerr), bucket)
+		return loi, b2ToObjectError(errors.Trace(lerr), bucket)
 	}
 	loi.IsTruncated = next != ""
 	loi.NextMarker = next
@@ -259,12 +325,12 @@ func (l *b2Objects) ListObjects(bucket string, prefix string, marker string, del
 		case "folder":
 			loi.Prefixes = append(loi.Prefixes, file.Name)
 		case "upload":
-			loi.Objects = append(loi.Objects, ObjectInfo{
+			loi.Objects = append(loi.Objects, minio.ObjectInfo{
 				Bucket:      bucket,
 				Name:        file.Name,
 				ModTime:     file.Timestamp,
 				Size:        file.Size,
-				ETag:        toS3ETag(file.Info.ID),
+				ETag:        minio.ToS3ETag(file.Info.ID),
 				ContentType: file.Info.ContentType,
 				UserDefined: file.Info.Info,
 			})
@@ -275,16 +341,15 @@ func (l *b2Objects) ListObjects(bucket string, prefix string, marker string, del
 
 // ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
 func (l *b2Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int,
-	fetchOwner bool, startAfter string) (loi ListObjectsV2Info, err error) {
+	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
 	// fetchOwner, startAfter are not supported and unused.
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
 		return loi, err
 	}
-	loi = ListObjectsV2Info{}
 	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, continuationToken, prefix, delimiter)
 	if lerr != nil {
-		return loi, b2ToObjectError(traceError(lerr), bucket)
+		return loi, b2ToObjectError(errors.Trace(lerr), bucket)
 	}
 	loi.IsTruncated = next != ""
 	loi.ContinuationToken = continuationToken
@@ -294,12 +359,12 @@ func (l *b2Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter s
 		case "folder":
 			loi.Prefixes = append(loi.Prefixes, file.Name)
 		case "upload":
-			loi.Objects = append(loi.Objects, ObjectInfo{
+			loi.Objects = append(loi.Objects, minio.ObjectInfo{
 				Bucket:      bucket,
 				Name:        file.Name,
 				ModTime:     file.Timestamp,
 				Size:        file.Size,
-				ETag:        toS3ETag(file.Info.ID),
+				ETag:        minio.ToS3ETag(file.Info.ID),
 				ContentType: file.Info.ContentType,
 				UserDefined: file.Info.Info,
 			})
@@ -321,38 +386,37 @@ func (l *b2Objects) GetObject(bucket string, object string, startOffset int64, l
 	}
 	reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length)
 	if err != nil {
-		return b2ToObjectError(traceError(err), bucket, object)
+		return b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 	defer reader.Close()
 	_, err = io.Copy(writer, reader)
-	return b2ToObjectError(traceError(err), bucket, object)
+	return b2ToObjectError(errors.Trace(err), bucket, object)
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo
-func (l *b2Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
+func (l *b2Objects) GetObjectInfo(bucket string, object string) (objInfo minio.ObjectInfo, err error) {
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
 		return objInfo, err
 	}
 	f, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
 	if err != nil {
-		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 	f.Close()
 	fi, err := bkt.File(f.ID, object).GetFileInfo(l.ctx)
 	if err != nil {
-		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 	}
-	objInfo = ObjectInfo{
+	return minio.ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
-		ETag:        toS3ETag(fi.ID),
+		ETag:        minio.ToS3ETag(fi.ID),
 		Size:        fi.Size,
 		ModTime:     fi.Timestamp,
 		ContentType: fi.ContentType,
 		UserDefined: fi.Info,
-	}
-	return objInfo, nil
+	}, nil
 }
 
 // In B2 - You must always include the X-Bz-Content-Sha1 header with
@@ -362,10 +426,8 @@ func (l *b2Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectI
 // (3) the string do_not_verify.
 // For more reference - https://www.backblaze.com/b2/docs/uploading.html
 //
-const (
-	sha1NoVerify = "do_not_verify"
-	sha1AtEOF    = "hex_digits_at_end"
-)
+// In our case we are going to use (2) option
+const sha1AtEOF = "hex_digits_at_end"
 
 // With the second option mentioned above, you append the 40-character hex sha1
 // to the end of the request body, immediately after the contents of the file
@@ -378,20 +440,20 @@ const (
 // Additionally this reader also verifies Hash encapsulated inside hash.Reader
 // at io.EOF if the verification failed we return an error and do not send
 // the content to server.
-func newB2Reader(r *h2.Reader, size int64) *B2Reader {
-	return &B2Reader{
+func newB2Reader(r *h2.Reader, size int64) *Reader {
+	return &Reader{
 		r:        r,
 		size:     size,
 		sha1Hash: sha1.New(),
 	}
 }
 
-// B2Reader - is a Reader wraps the hash.Reader which will emit out the sha1
+// Reader - is a Reader wraps the hash.Reader which will emit out the sha1
 // hex digits at io.EOF. It also means that your overall content size is
 // now original size + 40 bytes. Additionally this reader also verifies
 // Hash encapsulated inside hash.Reader at io.EOF if the verification
 // failed we return an error and do not send the content to server.
-type B2Reader struct {
+type Reader struct {
 	r        *h2.Reader
 	size     int64
 	sha1Hash hash.Hash
@@ -401,8 +463,8 @@ type B2Reader struct {
 }
 
 // Size - Returns the total size of Reader.
-func (nb *B2Reader) Size() int64 { return nb.size + 40 }
-func (nb *B2Reader) Read(p []byte) (int, error) {
+func (nb *Reader) Size() int64 { return nb.size + 40 }
+func (nb *Reader) Read(p []byte) (int, error) {
 	if nb.isEOF {
 		return nb.buf.Read(p)
 	}
@@ -421,8 +483,7 @@ func (nb *B2Reader) Read(p []byte) (int, error) {
 }
 
 // PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
-func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, metadata map[string]string) (ObjectInfo, error) {
-	var objInfo ObjectInfo
+func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
 		return objInfo, err
@@ -433,26 +494,26 @@ func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, met
 	var u *b2.URL
 	u, err = bkt.GetUploadURL(l.ctx)
 	if err != nil {
-		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 
 	hr := newB2Reader(data, data.Size())
 	var f *b2.File
 	f, err = u.UploadFile(l.ctx, hr, int(hr.Size()), object, contentType, sha1AtEOF, metadata)
 	if err != nil {
-		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 
 	var fi *b2.FileInfo
 	fi, err = f.GetFileInfo(l.ctx)
 	if err != nil {
-		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+		return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 
-	return ObjectInfo{
+	return minio.ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
-		ETag:        toS3ETag(fi.ID),
+		ETag:        minio.ToS3ETag(fi.ID),
 		Size:        fi.Size,
 		ModTime:     fi.Timestamp,
 		ContentType: fi.ContentType,
@@ -462,8 +523,8 @@ func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, met
 
 // CopyObject copies a blob from source container to destination container.
 func (l *b2Objects) CopyObject(srcBucket string, srcObject string, dstBucket string,
-	dstObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
-	return objInfo, traceError(NotImplemented{})
+	dstObject string, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+	return objInfo, errors.Trace(minio.NotImplemented{})
 }
 
 // DeleteObject deletes a blob in bucket
@@ -474,17 +535,17 @@ func (l *b2Objects) DeleteObject(bucket string, object string) error {
 	}
 	reader, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
 	if err != nil {
-		return b2ToObjectError(traceError(err), bucket, object)
+		return b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 	io.Copy(ioutil.Discard, reader)
 	reader.Close()
 	err = bkt.File(reader.ID, object).DeleteFileVersion(l.ctx)
-	return b2ToObjectError(traceError(err), bucket, object)
+	return b2ToObjectError(errors.Trace(err), bucket, object)
 }
 
 // ListMultipartUploads lists all multipart uploads.
 func (l *b2Objects) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string,
-	delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
+	delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {
 	// keyMarker, prefix, delimiter are all ignored, Backblaze B2 doesn't support any
 	// of these parameters only equivalent parameter is uploadIDMarker.
 	bkt, err := l.Bucket(bucket)
@@ -498,9 +559,9 @@ func (l *b2Objects) ListMultipartUploads(bucket string, prefix string, keyMarker
 	}
 	largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads)
 	if err != nil {
-		return lmi, b2ToObjectError(traceError(err), bucket)
+		return lmi, b2ToObjectError(errors.Trace(err), bucket)
 	}
-	lmi = ListMultipartsInfo{
+	lmi = minio.ListMultipartsInfo{
 		MaxUploads: maxUploads,
 	}
 	if nextMarker != "" {
@@ -508,7 +569,7 @@ func (l *b2Objects) ListMultipartUploads(bucket string, prefix string, keyMarker
 		lmi.NextUploadIDMarker = nextMarker
 	}
 	for _, largeFile := range largeFiles {
-		lmi.Uploads = append(lmi.Uploads, uploadMetadata{
+		lmi.Uploads = append(lmi.Uploads, minio.MultipartInfo{
 			Object:    largeFile.Name,
 			UploadID:  largeFile.ID,
 			Initiated: largeFile.Timestamp,
@@ -533,20 +594,14 @@ func (l *b2Objects) NewMultipartUpload(bucket string, object string, metadata ma
 	delete(metadata, "content-type")
 	lf, err := bkt.StartLargeFile(l.ctx, object, contentType, metadata)
 	if err != nil {
-		return uploadID, b2ToObjectError(traceError(err), bucket, object)
+		return uploadID, b2ToObjectError(errors.Trace(err), bucket, object)
 	}
 
 	return lf.ID, nil
 }
 
-// CopyObjectPart copy part of object to other bucket and object.
-func (l *b2Objects) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string,
-	uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
-	return PartInfo{}, traceError(NotImplemented{})
-}
-
 // PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
-func (l *b2Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi PartInfo, err error) {
+func (l *b2Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi minio.PartInfo, err error) {
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
 		return pi, err
@@ -554,30 +609,30 @@ func (l *b2Objects) PutObjectPart(bucket string, object string, uploadID string,
 
 	fc, err := bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
 	if err != nil {
-		return pi, b2ToObjectError(traceError(err), bucket, object, uploadID)
+		return pi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
 	}
 
 	hr := newB2Reader(data, data.Size())
 	sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
 	if err != nil {
-		return pi, b2ToObjectError(traceError(err), bucket, object, uploadID)
+		return pi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
 	}
 
-	return PartInfo{
+	return minio.PartInfo{
 		PartNumber:   partID,
-		LastModified: UTCNow(),
-		ETag:         toS3ETag(sha1),
+		LastModified: minio.UTCNow(),
+		ETag:         minio.ToS3ETag(sha1),
 		Size:         data.Size(),
 	}, nil
 }
 
 // ListObjectParts returns all object parts for specified object in specified bucket, uses B2's LargeFile upload API.
-func (l *b2Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) {
+func (l *b2Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, err error) {
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
 		return lpi, err
 	}
-	lpi = ListPartsInfo{
+	lpi = minio.ListPartsInfo{
 		Bucket:           bucket,
 		Object:           object,
 		UploadID:         uploadID,
@@ -588,16 +643,16 @@ func (l *b2Objects) ListObjectParts(bucket string, object string, uploadID strin
 	partNumberMarker++
 	partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
 	if err != nil {
-		return lpi, b2ToObjectError(traceError(err), bucket, object, uploadID)
+		return lpi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
 	}
 	if next != 0 {
 		lpi.IsTruncated = true
 		lpi.NextPartNumberMarker = next
 	}
 	for _, part := range partsList {
-		lpi.Parts = append(lpi.Parts, PartInfo{
+		lpi.Parts = append(lpi.Parts, minio.PartInfo{
 			PartNumber: part.Number,
-			ETag:       toS3ETag(part.SHA1),
+			ETag:       minio.ToS3ETag(part.SHA1),
 			Size:       part.Size,
 		})
 	}
@@ -611,11 +666,11 @@ func (l *b2Objects) AbortMultipartUpload(bucket string, object string, uploadID
 		return err
 	}
 	err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
-	return b2ToObjectError(traceError(err), bucket, object, uploadID)
+	return b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
 }
 
 // CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API.
-func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []completePart) (oi ObjectInfo, err error) {
+func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) {
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
 		return oi, err
@@ -625,7 +680,7 @@ func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, upload
 		// B2 requires contigous part numbers starting with 1, they do not support
 		// hand picking part numbers, we return an S3 compatible error instead.
 		if i+1 != uploadedPart.PartNumber {
-			return oi, b2ToObjectError(traceError(InvalidPart{}), bucket, object, uploadID)
+			return oi, b2ToObjectError(errors.Trace(minio.InvalidPart{}), bucket, object, uploadID)
 		}
 
 		// Trim "-1" suffix in ETag as PutObjectPart() treats B2 returned SHA1 as ETag.
@@ -633,7 +688,7 @@ func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, upload
 	}
 
 	if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
-		return oi, b2ToObjectError(traceError(err), bucket, object, uploadID)
+		return oi, b2ToObjectError(errors.Trace(err), bucket, object, uploadID)
 	}
 
 	return l.GetObjectInfo(bucket, object)
@@ -644,23 +699,23 @@ func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, upload
 // bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them.
 // Default is AllPrivate for all buckets.
 func (l *b2Objects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
-	var policies []BucketAccessPolicy
+	var policies []minio.BucketAccessPolicy
 
 	for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
-		policies = append(policies, BucketAccessPolicy{
+		policies = append(policies, minio.BucketAccessPolicy{
 			Prefix: prefix,
 			Policy: policy,
 		})
 	}
 	prefix := bucket + "/*" // For all objects inside the bucket.
 	if len(policies) != 1 {
-		return traceError(NotImplemented{})
+		return errors.Trace(minio.NotImplemented{})
 	}
 	if policies[0].Prefix != prefix {
-		return traceError(NotImplemented{})
+		return errors.Trace(minio.NotImplemented{})
 	}
 	if policies[0].Policy != policy.BucketPolicyReadOnly {
-		return traceError(NotImplemented{})
+		return errors.Trace(minio.NotImplemented{})
 	}
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
@@ -668,7 +723,7 @@ func (l *b2Objects) SetBucketPolicies(bucket string, policyInfo policy.BucketAcc
 	}
 	bkt.Type = bucketTypeReadOnly
 	_, err = bkt.Update(l.ctx)
-	return b2ToObjectError(traceError(err))
+	return b2ToObjectError(errors.Trace(err))
 }
 
 // GetBucketPolicies, returns the current bucketType from B2 backend and convert
@@ -686,7 +741,7 @@ func (l *b2Objects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy,
 	// bkt.Type can also be snapshot, but it is only allowed through B2 browser console,
 	// just return back as policy not found for all cases.
 	// CreateBucket always sets the value to allPrivate by default.
-	return policy.BucketAccessPolicy{}, traceError(PolicyNotFound{Bucket: bucket})
+	return policy.BucketAccessPolicy{}, errors.Trace(minio.PolicyNotFound{Bucket: bucket})
 }
 
 // DeleteBucketPolicies - resets the bucketType of bucket on B2 to 'allPrivate'.
@@ -697,5 +752,5 @@ func (l *b2Objects) DeleteBucketPolicies(bucket string) error {
 	}
 	bkt.Type = bucketTypePrivate
 	_, err = bkt.Update(l.ctx)
-	return b2ToObjectError(traceError(err))
+	return b2ToObjectError(errors.Trace(err))
 }

cmd/gateway/b2/gateway-b2_test.go

@@ -14,11 +14,17 @@
  * limitations under the License.
  */
 
-package cmd
+package b2
 
 import (
+	"fmt"
 	"net/http"
 	"testing"
+
+	b2 "github.com/minio/blazer/base"
+	"github.com/minio/minio/pkg/errors"
+
+	minio "github.com/minio/minio/cmd"
 )
 
 // Tests headerToObjectInfo
@@ -26,7 +32,7 @@ func TestHeaderToObjectInfo(t *testing.T) {
 	testCases := []struct {
 		bucket, object string
 		header         http.Header
-		objInfo        ObjectInfo
+		objInfo        minio.ObjectInfo
 	}{
 		{
 			bucket: "bucket",
@@ -38,7 +44,7 @@ func TestHeaderToObjectInfo(t *testing.T) {
 				"X-Bz-Info-X-Amz-Meta-1": []string{"test1"},
 				"X-Bz-File-Id":           []string{"xxxxx"},
 			},
-			objInfo: ObjectInfo{
+			objInfo: minio.ObjectInfo{
 				Bucket:      "bucket",
 				Name:        "object",
 				ContentType: "application/javascript",
@@ -102,3 +108,96 @@ func TestMkRange(t *testing.T) {
 		}
 	}
 }
+
+// Test b2 object error.
+func TestB2ObjectError(t *testing.T) {
+	testCases := []struct {
+		params      []string
+		b2Err       error
+		expectedErr error
+	}{
+		{
+			[]string{}, nil, nil,
+		},
+		{
+			[]string{}, fmt.Errorf("Not *Error"), fmt.Errorf("Not *Error"),
+		},
+		{
+			[]string{}, errors.Trace(fmt.Errorf("Non B2 Error")), fmt.Errorf("Non B2 Error"),
+		},
+		{
+			[]string{"bucket"}, errors.Trace(b2.Error{
+				StatusCode: 1,
+				Code:       "duplicate_bucket_name",
+			}), minio.BucketAlreadyOwnedByYou{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket"}, errors.Trace(b2.Error{
+				StatusCode: 1,
+				Code:       "bad_request",
+			}), minio.BucketNotFound{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket", "object"}, errors.Trace(b2.Error{
+				StatusCode: 1,
+				Code:       "bad_request",
+			}), minio.ObjectNameInvalid{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket"}, errors.Trace(b2.Error{
+				StatusCode: 1,
+				Code:       "bad_bucket_id",
+			}), minio.BucketNotFound{Bucket: "bucket"},
+		},
+		{
+			[]string{"bucket", "object"}, errors.Trace(b2.Error{
+				StatusCode: 1,
+				Code:       "file_not_present",
+			}), minio.ObjectNotFound{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket", "object"}, errors.Trace(b2.Error{
+				StatusCode: 1,
+				Code:       "not_found",
+			}), minio.ObjectNotFound{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket"}, errors.Trace(b2.Error{
+				StatusCode: 1,
+				Code:       "cannot_delete_non_empty_bucket",
+			}), minio.BucketNotEmpty{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket", "object", "uploadID"}, errors.Trace(b2.Error{
+				StatusCode: 1,
+				Message:    "No active upload for",
+			}), minio.InvalidUploadID{
+				UploadID: "uploadID",
+			},
+		},
+	}
+
+	for i, testCase := range testCases {
+		actualErr := b2ToObjectError(testCase.b2Err, testCase.params...)
+		if actualErr != nil {
+			if actualErr.Error() != testCase.expectedErr.Error() {
+				t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr)
+			}
+		}
+	}
+}

cmd/gateway/gateway.go

@@ -0,0 +1,29 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package gateway
+
+import (
+	// Import all gateways.
+	_ "github.com/minio/minio/cmd/gateway/azure"
+	_ "github.com/minio/minio/cmd/gateway/b2"
+	_ "github.com/minio/minio/cmd/gateway/gcs"
+	_ "github.com/minio/minio/cmd/gateway/manta"
+	_ "github.com/minio/minio/cmd/gateway/oss"
+	_ "github.com/minio/minio/cmd/gateway/s3"
+	_ "github.com/minio/minio/cmd/gateway/sia"
+	// Add your gateway here.
+)

cmd/gateway/gcs/gateway-gcs-anonymous.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package gcs
 
 import (
 	"fmt"
@@ -22,6 +22,10 @@ import (
 	"net/http"
 	"strconv"
 	"time"
+
+	"github.com/minio/minio/pkg/errors"
+
+	minio "github.com/minio/minio/cmd"
 )
 
 func toGCSPublicURL(bucket, object string) string {
@@ -32,7 +36,7 @@ func toGCSPublicURL(bucket, object string) string {
 func (l *gcsGateway) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
 	req, err := http.NewRequest("GET", toGCSPublicURL(bucket, object), nil)
 	if err != nil {
-		return gcsToObjectError(traceError(err), bucket, object)
+		return gcsToObjectError(errors.Trace(err), bucket, object)
 	}
 
 	if length > 0 && startOffset > 0 {
@@ -43,28 +47,28 @@ func (l *gcsGateway) AnonGetObject(bucket string, object string, startOffset int
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return gcsToObjectError(traceError(err), bucket, object)
+		return gcsToObjectError(errors.Trace(err), bucket, object)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
-		return gcsToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
+		return gcsToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
 	}
 
 	_, err = io.Copy(writer, resp.Body)
-	return gcsToObjectError(traceError(err), bucket, object)
+	return gcsToObjectError(errors.Trace(err), bucket, object)
 }
 
 // AnonGetObjectInfo - Get object info anonymously
-func (l *gcsGateway) AnonGetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
+func (l *gcsGateway) AnonGetObjectInfo(bucket string, object string) (objInfo minio.ObjectInfo, err error) {
 	resp, err := http.Head(toGCSPublicURL(bucket, object))
 	if err != nil {
-		return objInfo, gcsToObjectError(traceError(err), bucket, object)
+		return objInfo, gcsToObjectError(errors.Trace(err), bucket, object)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return objInfo, gcsToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
+		return objInfo, gcsToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
 	}
 
 	var contentLength int64
@@ -72,13 +76,13 @@ func (l *gcsGateway) AnonGetObjectInfo(bucket string, object string) (objInfo Ob
 	if contentLengthStr != "" {
 		contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
 		if err != nil {
-			return objInfo, gcsToObjectError(traceError(errUnexpected), bucket, object)
+			return objInfo, gcsToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object)
 		}
 	}
 
 	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
 	if err != nil {
-		return objInfo, traceError(err)
+		return objInfo, errors.Trace(err)
 	}
 
 	objInfo.ModTime = t
@@ -96,46 +100,46 @@ func (l *gcsGateway) AnonGetObjectInfo(bucket string, object string) (objInfo Ob
 }
 
 // AnonListObjects - List objects anonymously
-func (l *gcsGateway) AnonListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+func (l *gcsGateway) AnonListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (minio.ListObjectsInfo, error) {
 	result, err := l.anonClient.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		return ListObjectsInfo{}, s3ToObjectError(traceError(err), bucket)
+		return minio.ListObjectsInfo{}, minio.ErrorRespToObjectError(errors.Trace(err), bucket)
 	}
 
-	return fromMinioClientListBucketResult(bucket, result), nil
+	return minio.FromMinioClientListBucketResult(bucket, result), nil
 }
 
 // AnonListObjectsV2 - List objects in V2 mode, anonymously
-func (l *gcsGateway) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
+func (l *gcsGateway) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) {
 	// Request V1 List Object to the backend
 	result, err := l.anonClient.ListObjects(bucket, prefix, continuationToken, delimiter, maxKeys)
 	if err != nil {
-		return ListObjectsV2Info{}, s3ToObjectError(traceError(err), bucket)
+		return minio.ListObjectsV2Info{}, minio.ErrorRespToObjectError(errors.Trace(err), bucket)
 	}
 	// translate V1 Result to V2Info
-	return fromMinioClientListBucketResultToV2Info(bucket, result), nil
+	return minio.FromMinioClientListBucketResultToV2Info(bucket, result), nil
 }
 
 // AnonGetBucketInfo - Get bucket metadata anonymously.
-func (l *gcsGateway) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
+func (l *gcsGateway) AnonGetBucketInfo(bucket string) (bucketInfo minio.BucketInfo, err error) {
 	resp, err := http.Head(toGCSPublicURL(bucket, ""))
 	if err != nil {
-		return bucketInfo, gcsToObjectError(traceError(err))
+		return bucketInfo, gcsToObjectError(errors.Trace(err))
 	}
 
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return bucketInfo, gcsToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket)), bucket)
+		return bucketInfo, gcsToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket)), bucket)
 	}
 
 	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
 	if err != nil {
-		return bucketInfo, traceError(err)
+		return bucketInfo, errors.Trace(err)
 	}
 
 	// Last-Modified date being returned by GCS
-	return BucketInfo{
+	return minio.BucketInfo{
 		Name:    bucket,
 		Created: t,
 	}, nil