Fix free and total space reported in startup banner (#5419)

With storage class support, the free and total space
reported in Minio XL startup banner should be based on
totalDisks - standardClassParityDisks, instead of totalDisks/2.

fixes #5416

Dockerfile

@@ -6,6 +6,8 @@ ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
 ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key
 
 WORKDIR /go/src/github.com/minio/
 

Dockerfile.dev

@@ -6,6 +6,8 @@ ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
 ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key
 
 WORKDIR /go/src/github.com/minio/
 

Dockerfile.release

@@ -5,6 +5,8 @@ MAINTAINER Minio Inc <dev@minio.io>
 COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 
 ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key
 
 RUN \
      apk add --no-cache ca-certificates && \

README.md

@@ -99,6 +99,11 @@ Minio Server comes with an embedded web based object browser. Point your web bro
 ## Test using Minio Client `mc`
 `mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the Minio Client [Quickstart Guide](https://docs.minio.io/docs/minio-client-quickstart-guide) for further instructions.
 
+## Pre-existing data
+When deployed on a single drive, Minio server lets clients access any pre-existing data in the data directory. For example, if Minio is started with the command  `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.
+
+The above statement is also valid for all gateway backends.
+
 ## Explore Further
 - [Minio Erasure Code QuickStart Guide](https://docs.minio.io/docs/minio-erasure-code-quickstart-guide)
 - [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)

README_ZH.md

@@ -1,197 +0,0 @@
-# Minio 快速入门 [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio) [![Snap Status](https://build.snapcraft.io/badge/minio/minio.svg)](https://build.snapcraft.io/user/minio/minio)
-
-Minio是一个对象存储服务，基于Apache License v2.0协议. 它完全兼容亚马逊的S3云储存服务，非常适合于存储很多非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。 
-
-##  1. 下载
-
-Minio是一个非常轻量的服务,可以很简单的和其他应用的结合，类似 NodeJS, Redis 或者 MySQL.
-
-| Platform| Architecture | URL|
-| ----------| -------- | ------|
-|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio|
-|Apple OS X|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio|
-|Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe|
-|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
-
-### Homebrew 安装
-
-使用[Homebrew](http://brew.sh/) 来安装minio
- 
-```sh
-$ brew install minio
-$ minio --help
-```
-
-### 源码安装
-
-源码安装只针对开发者和一些高级用户，如果你还没有golang的环境，请安装golang官网安装[How to install Golang](https://docs.minio.io/docs/zh-CN/how-to-install-golang).
-
-```sh
-
-$ go get -u github.com/minio/minio
-
-
-```
-
-## 2. 运行Minio服务
-
-
-### GNU/Linux
-
- ```sh
-
-$ chmod +x minio
-$ ./minio --help
-$ ./minio server /data
-
-端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-AccessKey: USWUXHGYZQYFYFFIT3RE
-SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-区域:    us-east-1
-
-浏览器访问入口:
-   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-
-命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
-   $ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-
-对象操作API (兼容Amazon S3):
-   Go:         https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
-   Java:       https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
-   Python:     https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
-   JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
-
-```
-
-### OS X
-
-
- ```sh
-
-$ chmod 755 minio
-$ ./minio --help
-$ ./minio server /data
-
-端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-AccessKey: USWUXHGYZQYFYFFIT3RE
-SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-区域:    us-east-1
-
-浏览器访问入口:
-   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-
-命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
-   $ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-
-对象操作API (兼容Amazon S3):
-   Go:         https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
-   Java:       https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
-   Python:     https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
-   JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
-
-```
-
-### Microsoft Windows
-
-```sh
-
-C:\Users\Username\Downloads> minio.exe --help
-C:\Users\Username\Downloads> minio.exe server D:\Photos
-
-端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-AccessKey: USWUXHGYZQYFYFFIT3RE
-SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-区域:    us-east-1
-
-浏览器访问入口:
-   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-
-命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
-   $ mc.exe config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-
-对象操作API (兼容Amazon S3):
-   Go:         https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
-   Java:       https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
-   Python:     https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
-   JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
-
-
-```
-
-### Docker
-
-```sh
-
-$ docker pull minio/minio
-$ docker run -p 9000:9000 minio/minio
-
-```
-访问minio的docker入门指南获得更多内容 [here](https://docs.minio.io/docs/zh-CN/minio-docker-quickstart-guide)
-
-### FreeBSD
-
-```sh
-
-$ chmod 755 minio
-$ ./minio --help
-$ ./minio server /data
-
-端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-AccessKey: USWUXHGYZQYFYFFIT3RE
-SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-区域:    us-east-1
-
-浏览器访问入口:
-   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-
-命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
-   $ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-
-对象操作API (兼容Amazon S3):
-   Go:         https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
-   Java:       https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
-   Python:     https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
-   JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
-
-
-```
-请访问FreeBSD的官网指南获取更多详细信息[here](https://www.freebsd.org/doc/handbook/zfs-quickstart.html)
-
-## 3. 使用浏览器测试minio服务
-
-打开浏览器并输入 http://127.0.0.1:9000 查看在minio服务器上面的所有bucket
-
-![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true)
-
-
-## 4. 使用`mc`测试minio服务
-
-
-按照 [这个](https://docs.minio.io/docs/minio-client-quickstart-guide) 安装mc. 使用 `mc ls` 命令显示所有在minio服务上面的bucket.
-
-```sh
-
-$ mc ls myminio/
-[2015-08-05 08:13:22 IST]     0B andoria/
-[2015-08-05 06:14:26 IST]     0B deflector/
-[2015-08-05 08:13:11 IST]     0B ferenginar/
-[2016-03-08 14:56:35 IST]     0B jarjarbing/
-[2016-01-20 16:07:41 IST]     0B my.minio.io/
-
-```
-
-查看更多的例子请访问 [Minio Client Complete Guide](https://docs.minio.io/docs/zh-CN/minio-client-complete-guide).
-
-
-## 5. 更多内容
-
-- [Minio Erasure Code 快速入门](https://docs.minio.io/docs/zh-CN/minio-erasure-code-quickstart-guide)
-- [Minio Docker 快速入门](https://docs.minio.io/docs/zh-CN/minio-docker-quickstart-guide)
-- [使用`mc`测试 Minio Server](https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide)
-- [使用 `aws-cli` 测试 Minio Server](https://docs.minio.io/docs/zh-CN/aws-cli-with-minio)
-- [使用 `s3cmd` 测试 Minio Server](https://docs.minio.io/docs/zh-CN/s3cmd-with-minio)
-- [使用 `minio-go` SDK ce's测试 Minio Server](https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide)
-
-
-## 6. 给Minio项目贡献
-请按照Minio [贡献者指导手册](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)

browser/app/index.js

@@ -41,6 +41,8 @@ import _Login from './js/components/Login.js'
 import _Browse from './js/components/Browse.js'
 import fontAwesome from 'font-awesome/css/font-awesome.css'
 
+import MaterialDesignIconicFonts from 'material-design-iconic-font/dist/css/material-design-iconic-font.min.css'
+
 import Web from './js/web'
 window.Web = Web
 

browser/app/js/actions.js

@@ -24,6 +24,8 @@ export const SET_CURRENT_BUCKET = 'SET_CURRENT_BUCKET'
 export const SET_CURRENT_PATH = 'SET_CURRENT_PATH'
 export const SET_BUCKETS = 'SET_BUCKETS'
 export const ADD_BUCKET = 'ADD_BUCKET'
+export const REMOVE_BUCKET = 'REMOVE_BUCKET'
+export const SHOW_BUCKET_DROPDOWN = 'SHOW_BUCKET_DROPDOWN'
 export const SET_VISIBLE_BUCKETS = 'SET_VISIBLE_BUCKETS'
 export const SET_OBJECTS = 'SET_OBJECTS'
 export const APPEND_OBJECTS = 'APPEND_OBJECTS'
@@ -173,6 +175,27 @@ export const addBucket = bucket => {
   }
 }
 
+export const removeBucket = bucket => {
+  return {
+    type: REMOVE_BUCKET,
+    bucket
+  }
+}
+
+export const showBucketDropdown = bucket => {
+  return {
+    type: SHOW_BUCKET_DROPDOWN,
+    showBucketDropdown: true
+  }
+}
+
+export const hideBucketDropdown = bucket => {
+  return {
+    type: SHOW_BUCKET_DROPDOWN,
+    showBucketDropdown: false
+  }
+}
+
 export const showMakeBucketModal = () => {
   return {
     type: SHOW_MAKEBUCKET_MODAL,
@@ -314,6 +337,31 @@ export const selectBucket = (newCurrentBucket, prefix) => {
   }
 }
 
+export const deleteBucket = (bucket) => {
+  return (dispatch, getState) => {
+    // DeleteBucket() RPC call will ONLY delete a bucket if it is empty of
+    // objects. This means a call can just be sent, as it is entirely reversable
+    // and won't do any permanent damage.
+    web.DeleteBucket({
+      bucketName: bucket
+    })
+      .then(() => {
+        dispatch(showAlert({
+          type: 'info',
+          message: `Bucket '${bucket}' has been deleted.`
+        }))
+        dispatch(removeBucket(bucket))
+      })
+      .catch(err => {
+        let message = err.message
+        dispatch(showAlert({
+          type: 'danger',
+          message: message
+        }))
+      })
+  }
+}
+
 export const listObjects = () => {
   return (dispatch, getState) => {
     const {buckets, currentBucket, currentPath, marker, objects, istruncated, web} = getState()
@@ -352,7 +400,6 @@ export const listObjects = () => {
 export const selectPrefix = prefix => {
   return (dispatch, getState) => {
     const {currentBucket, web} = getState()
-    dispatch(resetObjects())
     dispatch(setLoadPath(prefix))
     web.ListObjects({
       bucketName: currentBucket,
@@ -367,6 +414,7 @@ export const selectPrefix = prefix => {
           object.name = object.name.replace(`${prefix}`, '');
           return object
         })
+        dispatch(resetObjects())
         dispatch(appendObjects(
           objects,
           res.nextmarker,

browser/app/js/components/Browse.js

@@ -210,9 +210,19 @@ export default class Browse extends React.Component {
     dispatch(actions.hideAbout())
   }
 
+  toggleBucketDropdown(e) {
+    const {dispatch, showBucketDropdown} = this.props
+    if (showBucketDropdown) {
+      dispatch(actions.hideBucketDropdown())
+    } else {
+      dispatch(actions.showBucketDropdown())
+    }
+  }
+
   showBucketPolicy(e) {
     e.preventDefault()
     const {dispatch} = this.props
+    this.toggleBucketDropdown(e)
     dispatch(actions.showBucketPolicy())
   }
 
@@ -222,10 +232,17 @@ export default class Browse extends React.Component {
     dispatch(actions.hideBucketPolicy())
   }
 
+  deleteBucket(e, bucket) {
+    e.preventDefault()
+    const {dispatch} = this.props
+    this.toggleBucketDropdown(e)
+    dispatch(actions.deleteBucket(bucket))
+    browserHistory.push(`${minioBrowserPrefix}/`)
+  }
+
   uploadFile(e) {
     e.preventDefault()
-    const {dispatch, buckets} = this.props
-
+    const {dispatch, buckets, currentBucket} = this.props
     if (buckets.length === 0) {
       dispatch(actions.showAlert({
         type: 'danger',
@@ -233,6 +250,13 @@ export default class Browse extends React.Component {
       }))
       return
     }
+    if (currentBucket === '') {
+      dispatch(actions.showAlert({
+        type: 'danger',
+        message: "Please choose a bucket before trying to upload files."
+      }))
+      return
+    }
     let file = e.target.files[0]
     e.target.value = null
     this.xhr = new XMLHttpRequest()
@@ -577,7 +601,9 @@ export default class Browse extends React.Component {
         <SideBar searchBuckets={ this.searchBuckets.bind(this) }
           selectBucket={ this.selectBucket.bind(this) }
           clickOutside={ this.hideSidebar.bind(this) }
-          showPolicy={ this.showBucketPolicy.bind(this) } />
+          showPolicy={ this.showBucketPolicy.bind(this) }
+          deleteBucket={ this.deleteBucket.bind(this) }
+          toggleBucketDropdown={ this.toggleBucketDropdown.bind(this) } />
         <div className="fe-body">
           <div className={ 'list-actions' + (classNames({
                              ' list-actions-toggled': checkedObjects.length > 0

browser/app/js/components/SideBar.js

@@ -21,8 +21,9 @@ import Scrollbars from 'react-custom-scrollbars/lib/Scrollbars'
 import connect from 'react-redux/lib/components/connect'
 
 import logo from '../../img/logo.svg'
+import Dropdown from 'react-bootstrap/lib/Dropdown'
 
-let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchBuckets, sidebarStatus, clickOutside, showPolicy}) => {
+let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchBuckets, sidebarStatus, clickOutside, showPolicy, deleteBucket, toggleBucketDropdown, showBucketDropdown}) => {
 
   const list = visibleBuckets.map((bucket, i) => {
     return <li className={ classNames({
@@ -33,7 +34,19 @@ let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchB
                                     }) }>
                { bucket }
              </a>
-             <i className="fesli-trigger" onClick={ showPolicy }></i>
+             <Dropdown open={bucket === currentBucket && showBucketDropdown} onToggle={toggleBucketDropdown} className="bucket-dropdown" id="bucket-dropdown">
+               <Dropdown.Toggle noCaret>
+                 <i className="zmdi zmdi-more-vert" />
+               </Dropdown.Toggle>
+               <Dropdown.Menu className="dropdown-menu-right">
+                 <li>
+                   <a onClick={ showPolicy }>Edit policy</a>
+                 </li>
+                 <li>
+                   <a onClick={ (e) => deleteBucket(e, bucket) }>Delete</a>
+                 </li>
+               </Dropdown.Menu>
+             </Dropdown>
            </li>
   })
 
@@ -80,6 +93,7 @@ export default connect(state => {
     visibleBuckets: state.visibleBuckets,
     loadBucket: state.loadBucket,
     currentBucket: state.currentBucket,
+    showBucketDropdown: state.showBucketDropdown,
     sidebarStatus: state.sidebarStatus
   }
 })(SideBar)

browser/app/js/reducers.js

@@ -25,6 +25,7 @@ export default (state = {
     storageInfo: {},
     serverInfo: {},
     currentBucket: '',
+    showBucketDropdown: false,
     currentPath: '',
     showMakeBucketModal: false,
     uploads: {},
@@ -71,6 +72,14 @@ export default (state = {
       newState.buckets = [action.bucket, ...newState.buckets]
       newState.visibleBuckets = [action.bucket, ...newState.visibleBuckets]
       break
+    case actions.REMOVE_BUCKET:
+      newState.buckets = newState.buckets.filter(bucket => bucket != action.bucket)
+      newState.visibleBuckets = newState.visibleBuckets.filter(bucket => bucket != action.bucket)
+      newState.currentBucket = ""
+      break
+    case actions.SHOW_BUCKET_DROPDOWN:
+      newState.showBucketDropdown = action.showBucketDropdown
+      break
     case actions.SET_VISIBLE_BUCKETS:
       newState.visibleBuckets = action.visibleBuckets
       break

browser/app/js/web.js

@@ -87,6 +87,9 @@ export default class Web {
   MakeBucket(args) {
     return this.makeCall('MakeBucket', args)
   }
+  DeleteBucket(args) {
+    return this.makeCall('DeleteBucket', args)
+  }
   ListObjects(args) {
     return this.makeCall('ListObjects', args)
   }

browser/app/less/inc/list.less

@@ -108,7 +108,7 @@ div.fesl-row {
     &[data-type=zip] { .list-type(#427089, '\f1c6'); }
     &[data-type=audio] { .list-type(#009688, '\f1c7'); }
     &[data-type=code] { .list-type(#997867, "\f1c9"); }
-    &[data-type=excel] { .list-type(#f1c3, '\f1c3'); }
+    &[data-type=excel] { .list-type(#64c866, '\f1c3'); }
     &[data-type=image] { .list-type(#f06292, '\f1c5'); }
     &[data-type=video] { .list-type(#f8c363, '\f1c8'); }
     &[data-type=other] { .list-type(#afafaf, '\f016'); }

browser/app/less/inc/sidebar.less

@@ -111,7 +111,7 @@
 		}
 
 		&:hover {
-			.fesli-trigger {
+			.bucket-dropdown .dropdown-toggle {
 				.opacity(0.6);
 
 				&:hover {
@@ -132,17 +132,40 @@
 	}
 }
 
-.fesli-trigger {
-	.opacity(0);
-	.transition(all);
-	.transition-duration(200ms);
+/* Dropdown */
+.bucket-dropdown {
 	position: absolute;
-	top: 0;
-	right: 0;
+	top: 0px;
+	right: 0px;
 	width: 35px;
 	height: 100%;
 	cursor: pointer;
-	background: url(../../img/more-h-light.svg) no-repeat top 20px left;
+	color: @white;
+
+	.dropdown-toggle {
+		.opacity(0);
+		.transition(all);
+		.transition-duration(200ms);
+		font-size: 20px;
+		background-color: transparent;
+	}
+
+	.dropdown-menu-right {
+		padding: 15px 0;
+		margin-top: -1px;
+		& li {
+			a:before {
+				content: none;
+				-webkit-box-shadow: @dropdown-shadow;
+				box-shadow: @dropdown-shadow;	
+			}
+			&:not(.active):hover {
+				& > a {
+					color: @dropdown-link-hover-color;
+				}
+			}
+		}
+	}
 }
 
 /* Scrollbar */

browser/package.json

@@ -60,6 +60,7 @@
     "humanize": "0.0.9",
     "json-loader": "^0.5.4",
     "local-storage-fallback": "^1.3.0",
+    "material-design-iconic-font": "^2.2.0",
     "mime-db": "^1.25.0",
     "mime-types": "^2.1.13",
     "moment": "^2.15.1",

cmd/admin-handlers.go

@@ -21,10 +21,10 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
-	"path"
 	"strconv"
 	"sync"
 	"time"
@@ -35,6 +35,8 @@ import (
 const (
 	minioAdminOpHeader   = "X-Minio-Operation"
 	minioConfigTmpFormat = "config-%s.json"
+
+	maxConfigJSONSize = 256 * 1024 // 256KiB
 )
 
 // Type-safe query params.
@@ -42,18 +44,14 @@ type mgmtQueryKey string
 
 // Only valid query params for mgmt admin APIs.
 const (
-	mgmtBucket         mgmtQueryKey = "bucket"
-	mgmtObject         mgmtQueryKey = "object"
-	mgmtPrefix         mgmtQueryKey = "prefix"
-	mgmtLockDuration   mgmtQueryKey = "duration"
-	mgmtDelimiter      mgmtQueryKey = "delimiter"
-	mgmtMarker         mgmtQueryKey = "marker"
-	mgmtKeyMarker      mgmtQueryKey = "key-marker"
-	mgmtMaxKey         mgmtQueryKey = "max-key"
-	mgmtDryRun         mgmtQueryKey = "dry-run"
-	mgmtUploadIDMarker mgmtQueryKey = "upload-id-marker"
-	mgmtMaxUploads     mgmtQueryKey = "max-uploads"
-	mgmtUploadID       mgmtQueryKey = "upload-id"
+	mgmtBucket       mgmtQueryKey = "bucket"
+	mgmtObject       mgmtQueryKey = "object"
+	mgmtPrefix       mgmtQueryKey = "prefix"
+	mgmtLockDuration mgmtQueryKey = "duration"
+	mgmtDelimiter    mgmtQueryKey = "delimiter"
+	mgmtMarker       mgmtQueryKey = "marker"
+	mgmtMaxKey       mgmtQueryKey = "max-key"
+	mgmtDryRun       mgmtQueryKey = "dry-run"
 )
 
 // ServerVersion - server version
@@ -74,7 +72,7 @@ type ServerStatus struct {
 // Fetches server status information like total disk space available
 // to use, online disks, offline disks and quorum threshold.
 func (adminAPI adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
@@ -116,7 +114,7 @@ func (adminAPI adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *
 // Restarts minio server gracefully. In a distributed setup,  restarts
 // all the servers in the cluster.
 func (adminAPI adminAPIHandlers) ServiceRestartHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
@@ -141,7 +139,7 @@ type setCredsReq struct {
 // in the cluster.
 func (adminAPI adminAPIHandlers) ServiceCredentialsHandler(w http.ResponseWriter, r *http.Request) {
 	// Authenticate request
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
@@ -259,7 +257,7 @@ type ServerInfo struct {
 // Get server information
 func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	// Authenticate request
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
@@ -343,7 +341,7 @@ func validateLockQueryParams(vars url.Values) (string, string, time.Duration, AP
 // ---------
 // Lists locks held on a given bucket, prefix and duration it was held for.
 func (adminAPI adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
@@ -385,7 +383,7 @@ func (adminAPI adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http
 // ---------
 // Clear locks held on a given bucket, prefix and duration it was held for.
 func (adminAPI adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
@@ -414,65 +412,12 @@ func (adminAPI adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *htt
 		errorIf(err, "Failed to marshal lock information into json.")
 		return
 	}
-
-	// Remove lock matching bucket/prefix held longer than duration.
-	for _, volLock := range volLocks {
-		globalNSMutex.ForceUnlock(volLock.Bucket, volLock.Object)
-	}
+	newObjectLayerFn().ClearLocks(volLocks)
 
 	// Reply with list of locks cleared, as json.
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-// ListUploadsHealHandler - similar to listObjectsHealHandler
-// GET
-// /?heal&bucket=mybucket&prefix=myprefix&key-marker=mymarker&upload-id-marker=myuploadid&delimiter=mydelimiter&max-uploads=1000
-// - bucket is mandatory query parameter
-// - rest are optional query parameters List upto maxKey objects that
-// need healing in a given bucket matching the given prefix.
-func (adminAPI adminAPIHandlers) ListUploadsHealHandler(w http.ResponseWriter, r *http.Request) {
-	// Get object layer instance.
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
-		return
-	}
-
-	// Validate request signature.
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
-	if adminAPIErr != ErrNone {
-		writeErrorResponse(w, adminAPIErr, r.URL)
-		return
-	}
-
-	// Validate query params.
-	vars := r.URL.Query()
-	bucket := vars.Get(string(mgmtBucket))
-	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
-
-	if err := checkListMultipartArgs(bucket, prefix, keyMarker, uploadIDMarker, delimiter, objLayer); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-
-	if maxUploads <= 0 || maxUploads > maxUploadsList {
-		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL)
-		return
-	}
-
-	// Get the list objects to be healed.
-	listMultipartInfos, err := objLayer.ListUploadsHeal(bucket, prefix,
-		keyMarker, uploadIDMarker, delimiter, maxUploads)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-
-	listResponse := generateListMultipartUploadsResponse(bucket, listMultipartInfos)
-	// Write success response.
-	writeSuccessResponseXML(w, encodeResponse(listResponse))
-}
-
 // extractListObjectsHealQuery - Validates query params for heal objects list management API.
 func extractListObjectsHealQuery(vars url.Values) (string, string, string, string, int, APIErrorCode) {
 	bucket := vars.Get(string(mgmtBucket))
@@ -522,12 +467,18 @@ func (adminAPI adminAPIHandlers) ListObjectsHealHandler(w http.ResponseWriter, r
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
 	}
 
+	// Check if this setup has an erasure coded backend.
+	if !globalIsXL {
+		writeErrorResponse(w, ErrHealNotImplemented, r.URL)
+		return
+	}
+
 	// Validate query params.
 	vars := r.URL.Query()
 	bucket, prefix, marker, delimiter, maxKey, adminAPIErr := extractListObjectsHealQuery(vars)
@@ -558,12 +509,18 @@ func (adminAPI adminAPIHandlers) ListBucketsHealHandler(w http.ResponseWriter, r
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
 	}
 
+	// Check if this setup has an erasure coded backend.
+	if !globalIsXL {
+		writeErrorResponse(w, ErrHealNotImplemented, r.URL)
+		return
+	}
+
 	// Get the list buckets to be healed.
 	bucketsInfo, err := objLayer.ListBucketsHeal()
 	if err != nil {
@@ -589,12 +546,18 @@ func (adminAPI adminAPIHandlers) HealBucketHandler(w http.ResponseWriter, r *htt
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
 	}
 
+	// Check if this setup has an erasure coded backend.
+	if !globalIsXL {
+		writeErrorResponse(w, ErrHealNotImplemented, r.URL)
+		return
+	}
+
 	// Validate bucket name and check if it exists.
 	vars := r.URL.Query()
 	bucket := vars.Get(string(mgmtBucket))
@@ -678,12 +641,18 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
 	}
 
+	// Check if this setup has an erasure coded backend.
+	if !globalIsXL {
+		writeErrorResponse(w, ErrHealNotImplemented, r.URL)
+		return
+	}
+
 	vars := r.URL.Query()
 	bucket := vars.Get(string(mgmtBucket))
 	object := vars.Get(string(mgmtObject))
@@ -723,81 +692,6 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-// HealUploadHandler - POST /?heal&bucket=mybucket&object=myobject&upload-id=myuploadID&dry-run
-// - x-minio-operation = upload
-// - bucket, object and upload-id are mandatory query parameters
-// Heal a given upload, if present.
-func (adminAPI adminAPIHandlers) HealUploadHandler(w http.ResponseWriter, r *http.Request) {
-	// Get object layer instance.
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
-		return
-	}
-
-	// Validate request signature.
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
-	if adminAPIErr != ErrNone {
-		writeErrorResponse(w, adminAPIErr, r.URL)
-		return
-	}
-
-	vars := r.URL.Query()
-	bucket := vars.Get(string(mgmtBucket))
-	object := vars.Get(string(mgmtObject))
-	uploadID := vars.Get(string(mgmtUploadID))
-	uploadObj := path.Join(bucket, object, uploadID)
-
-	// Validate bucket and object names as supplied via query
-	// parameters.
-	if err := checkBucketAndObjectNames(bucket, object); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-
-	// Validate the bucket and object w.r.t backend representation
-	// of an upload.
-	if err := checkBucketAndObjectNames(minioMetaMultipartBucket,
-		uploadObj); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-
-	// Check if upload exists.
-	if _, err := objLayer.GetObjectInfo(minioMetaMultipartBucket,
-		uploadObj); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-
-	// if dry-run is set in query params then perform validations
-	// and return success.
-	if isDryRun(vars) {
-		writeSuccessResponseHeadersOnly(w)
-		return
-	}
-
-	//We are able to use HealObject for healing an upload since an
-	//ongoing upload has the same backend representation as an
-	//object.  The 'object' corresponding to a given bucket,
-	//object and uploadID is
-	//.minio.sys/multipart/bucket/object/uploadID.
-	numOfflineDisks, numHealedDisks, err := objLayer.HealObject(minioMetaMultipartBucket, uploadObj)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-
-	jsonBytes, err := json.Marshal(newHealResult(numHealedDisks, numOfflineDisks))
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-
-	// Return 200 on success.
-	writeSuccessResponseJSON(w, jsonBytes)
-}
-
 // HealFormatHandler - POST /?heal&dry-run
 // - x-minio-operation = format
 // - bucket and object are both mandatory query parameters
@@ -811,7 +705,7 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
@@ -821,7 +715,7 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
 	// heal-format is only applicable to single node XL and
 	// distributed XL setup.
 	if !globalIsXL {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(w, ErrHealNotImplemented, r.URL)
 		return
 	}
 
@@ -878,7 +772,7 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
 // Get config.json of this minio setup.
 func (adminAPI adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	// Validate request signature.
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
@@ -971,29 +865,46 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponse(w, adminAPIErr, r.URL)
 		return
 	}
 
 	// Read configuration bytes from request body.
-	configBytes, err := ioutil.ReadAll(r.Body)
-	if err != nil {
+	configBuf := make([]byte, maxConfigJSONSize+1)
+	n, err := io.ReadFull(r.Body, configBuf)
+	if err == nil {
+		// More than maxConfigSize bytes were available
+		writeErrorResponse(w, ErrAdminConfigTooLarge, r.URL)
+		return
+	}
+	if err != io.ErrUnexpectedEOF {
 		errorIf(err, "Failed to read config from request body.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
+	configBytes := configBuf[:n]
+
+	// Validate JSON provided in the request body: check the
+	// client has not sent JSON objects with duplicate keys.
+	if err = checkDupJSONKeys(string(configBytes)); err != nil {
+		errorIf(err, "config contains duplicate JSON entries.")
+		writeErrorResponse(w, ErrAdminConfigBadJSON, r.URL)
+		return
+	}
+
 	var config serverConfig
 	err = json.Unmarshal(configBytes, &config)
-
 	if err != nil {
-		errorIf(err, "Failed to unmarshal config from request body.")
+		errorIf(err, "Failed to unmarshal JSON configuration", err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
+	// If credentials for the server are provided via environment,
+	// then credentials in the provided configuration must match.
 	if globalIsEnvCreds {
 		creds := globalServerConfig.GetCredential()
 		if config.Credential.AccessKey != creds.AccessKey ||

cmd/admin-handlers_test.go

@@ -27,16 +27,16 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
-	"reflect"
+	"strings"
 	"testing"
-	"time"
 
 	router "github.com/gorilla/mux"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/errors"
 )
 
-var configJSON = []byte(`{
+var (
+	configJSON = []byte(`{
 	"version": "13",
 	"credential": {
 		"accessKey": "minio",
@@ -131,6 +131,7 @@ var configJSON = []byte(`{
 		}
 	}
 }`)
+)
 
 // adminXLTestBed - encapsulates subsystems that need to be setup for
 // admin-handler unit tests.
@@ -193,7 +194,7 @@ func (atb *adminXLTestBed) TearDown() {
 // initTestObjLayer - Helper function to initialize an XL-based object
 // layer and set globalObjectAPI.
 func initTestXLObjLayer() (ObjectLayer, []string, error) {
-	objLayer, xlDirs, xlErr := prepareXL()
+	objLayer, xlDirs, xlErr := prepareXL16()
 	if xlErr != nil {
 		return nil, nil, xlErr
 	}
@@ -1048,140 +1049,6 @@ func buildAdminRequest(queryVal url.Values, opHdr, method string,
 	return req, nil
 }
 
-// mkHealUploadQuery - helper to build HealUploadHandler query string.
-func mkHealUploadQuery(bucket, object, uploadID, dryRun string) url.Values {
-	queryVal := url.Values{}
-	queryVal.Set(string(mgmtBucket), bucket)
-	queryVal.Set(string(mgmtObject), object)
-	queryVal.Set(string(mgmtUploadID), uploadID)
-	queryVal.Set("heal", "")
-	queryVal.Set(string(mgmtDryRun), dryRun)
-	return queryVal
-}
-
-// TestHealUploadHandler - test for HealUploadHandler.
-func TestHealUploadHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// Create an object myobject under bucket mybucket.
-	bucketName := "mybucket"
-	objName := "myobject"
-	err = adminTestBed.objLayer.MakeBucketWithLocation(bucketName, "")
-	if err != nil {
-		t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
-	}
-
-	// Create a new multipart upload.
-	uploadID, err := adminTestBed.objLayer.NewMultipartUpload(bucketName, objName, nil)
-	if err != nil {
-		t.Fatalf("Failed to create a new multipart upload %s/%s - %v",
-			bucketName, objName, err)
-	}
-
-	// Upload a part.
-	partID := 1
-	_, err = adminTestBed.objLayer.PutObjectPart(bucketName, objName, uploadID,
-		partID, mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""))
-	if err != nil {
-		t.Fatalf("Failed to upload part %d of %s/%s - %v", partID,
-			bucketName, objName, err)
-	}
-
-	testCases := []struct {
-		bucket     string
-		object     string
-		dryrun     string
-		statusCode int
-	}{
-		// 1. Valid test case.
-		{
-			bucket:     bucketName,
-			object:     objName,
-			statusCode: http.StatusOK,
-		},
-		// 2. Invalid bucket name.
-		{
-			bucket:     `invalid\\Bucket`,
-			object:     "myobject",
-			statusCode: http.StatusBadRequest,
-		},
-		// 3. Bucket not found.
-		{
-			bucket:     "bucketnotfound",
-			object:     "myobject",
-			statusCode: http.StatusNotFound,
-		},
-		// 4. Invalid object name.
-		{
-			bucket:     bucketName,
-			object:     `invalid\\Object`,
-			statusCode: http.StatusBadRequest,
-		},
-		// 5. Object not found.
-		{
-			bucket:     bucketName,
-			object:     "objectnotfound",
-			statusCode: http.StatusNotFound,
-		},
-		// 6. Valid test case with dry-run.
-		{
-			bucket:     bucketName,
-			object:     objName,
-			dryrun:     "yes",
-			statusCode: http.StatusOK,
-		},
-	}
-	for i, test := range testCases {
-		// Prepare query params.
-		queryVal := mkHealUploadQuery(test.bucket, test.object, uploadID, test.dryrun)
-		req, err1 := buildAdminRequest(queryVal, "upload", http.MethodPost, 0, nil)
-		if err1 != nil {
-			t.Fatalf("Test %d - Failed to construct heal object request - %v", i+1, err1)
-		}
-		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
-		if test.statusCode != rec.Code {
-			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
-		}
-	}
-
-	sample := testCases[0]
-	// Modify authorization header after signing to test signature
-	// mismatch handling.
-	queryVal := mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun)
-	req, err := buildAdminRequest(queryVal, "upload", "POST", 0, nil)
-	if err != nil {
-		t.Fatalf("Failed to construct heal object request - %v", err)
-	}
-
-	// Set x-amz-date to a date different than time of signing.
-	req.Header.Set("x-amz-date", time.Time{}.Format(iso8601Format))
-
-	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
-	if rec.Code != http.StatusForbidden {
-		t.Errorf("Expected %d but received %d", http.StatusBadRequest, rec.Code)
-	}
-
-	// Set objectAPI to nil to test Server not initialized case.
-	resetGlobalObjectAPI()
-	queryVal = mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun)
-	req, err = buildAdminRequest(queryVal, "upload", "POST", 0, nil)
-	if err != nil {
-		t.Fatalf("Failed to construct heal object request - %v", err)
-	}
-
-	rec = httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
-	if rec.Code != http.StatusServiceUnavailable {
-		t.Errorf("Expected %d but received %d", http.StatusServiceUnavailable, rec.Code)
-	}
-}
-
 // TestHealFormatHandler - test for HealFormatHandler.
 func TestHealFormatHandler(t *testing.T) {
 	adminTestBed, err := prepareAdminXLTestBed()
@@ -1257,7 +1124,7 @@ func TestSetConfigHandler(t *testing.T) {
 	req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(configJSON)),
 		bytes.NewReader(configJSON))
 	if err != nil {
-		t.Fatalf("Failed to construct get-config object request - %v", err)
+		t.Fatalf("Failed to construct set-config object request - %v", err)
 	}
 
 	rec := httptest.NewRecorder()
@@ -1275,6 +1142,44 @@ func TestSetConfigHandler(t *testing.T) {
 	if !result.Status {
 		t.Error("Expected set-config to succeed, but failed")
 	}
+
+	// Check that a very large config file returns an error.
+	{
+		// Make a large enough config string
+		invalidCfg := []byte(strings.Repeat("A", maxConfigJSONSize+1))
+		req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(invalidCfg)),
+			bytes.NewReader(invalidCfg))
+		if err != nil {
+			t.Fatalf("Failed to construct set-config object request - %v", err)
+		}
+
+		rec := httptest.NewRecorder()
+		adminTestBed.mux.ServeHTTP(rec, req)
+		respBody := string(rec.Body.Bytes())
+		if rec.Code != http.StatusBadRequest ||
+			!strings.Contains(respBody, "Configuration data provided exceeds the allowed maximum of") {
+			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
+		}
+	}
+
+	// Check that a config with duplicate keys in an object return
+	// error.
+	{
+		invalidCfg := append(configJSON[:len(configJSON)-1], []byte(`, "version": "15"}`)...)
+		req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(invalidCfg)),
+			bytes.NewReader(invalidCfg))
+		if err != nil {
+			t.Fatalf("Failed to construct set-config object request - %v", err)
+		}
+
+		rec := httptest.NewRecorder()
+		adminTestBed.mux.ServeHTTP(rec, req)
+		respBody := string(rec.Body.Bytes())
+		if rec.Code != http.StatusBadRequest ||
+			!strings.Contains(respBody, "JSON configuration provided has objects with duplicate keys") {
+			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
+		}
+	}
 }
 
 func TestAdminServerInfo(t *testing.T) {
@@ -1436,172 +1341,6 @@ func TestWriteSetConfigResponse(t *testing.T) {
 	}
 }
 
-// mkListUploadsHealQuery - helper function to construct query values for
-// listUploadsHeal.
-func mkListUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values {
-
-	queryVal := make(url.Values)
-	queryVal.Set("heal", "")
-	queryVal.Set(string(mgmtBucket), bucket)
-	queryVal.Set(string(mgmtPrefix), prefix)
-	queryVal.Set(string(mgmtKeyMarker), keyMarker)
-	queryVal.Set(string(mgmtUploadIDMarker), uploadIDMarker)
-	queryVal.Set(string(mgmtDelimiter), delimiter)
-	queryVal.Set(string(mgmtMaxUploads), maxUploadsStr)
-	return queryVal
-}
-
-func TestListHealUploadsHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
-	if err != nil {
-		t.Fatalf("Failed to make bucket - %v", err)
-	}
-
-	// Delete bucket after running all test cases.
-	defer adminTestBed.objLayer.DeleteBucket("mybucket")
-
-	testCases := []struct {
-		bucket       string
-		prefix       string
-		keyMarker    string
-		delimiter    string
-		maxKeys      string
-		statusCode   int
-		expectedResp ListMultipartUploadsResponse
-	}{
-		// 1. Valid params.
-		{
-			bucket:     "mybucket",
-			prefix:     "prefix",
-			keyMarker:  "prefix11",
-			delimiter:  "/",
-			maxKeys:    "10",
-			statusCode: http.StatusOK,
-			expectedResp: ListMultipartUploadsResponse{
-				XMLName:    xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListMultipartUploadsResult"},
-				Bucket:     "mybucket",
-				KeyMarker:  "prefix11",
-				Delimiter:  "/",
-				Prefix:     "prefix",
-				MaxUploads: 10,
-			},
-		},
-		// 2. Valid params with empty prefix.
-		{
-			bucket:     "mybucket",
-			prefix:     "",
-			keyMarker:  "",
-			delimiter:  "/",
-			maxKeys:    "10",
-			statusCode: http.StatusOK,
-			expectedResp: ListMultipartUploadsResponse{
-				XMLName:    xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListMultipartUploadsResult"},
-				Bucket:     "mybucket",
-				KeyMarker:  "",
-				Delimiter:  "/",
-				Prefix:     "",
-				MaxUploads: 10,
-			},
-		},
-		// 3. Invalid params with invalid bucket.
-		{
-			bucket:     `invalid\\Bucket`,
-			prefix:     "prefix",
-			keyMarker:  "prefix11",
-			delimiter:  "/",
-			maxKeys:    "10",
-			statusCode: getAPIError(ErrInvalidBucketName).HTTPStatusCode,
-		},
-		// 4. Invalid params with invalid prefix.
-		{
-			bucket:     "mybucket",
-			prefix:     `invalid\\Prefix`,
-			keyMarker:  "prefix11",
-			delimiter:  "/",
-			maxKeys:    "10",
-			statusCode: getAPIError(ErrInvalidObjectName).HTTPStatusCode,
-		},
-		// 5. Invalid params with invalid maxKeys.
-		{
-			bucket:     "mybucket",
-			prefix:     "prefix",
-			keyMarker:  "prefix11",
-			delimiter:  "/",
-			maxKeys:    "-1",
-			statusCode: getAPIError(ErrInvalidMaxUploads).HTTPStatusCode,
-		},
-		// 6. Invalid params with unsupported prefix marker combination.
-		{
-			bucket:     "mybucket",
-			prefix:     "prefix",
-			keyMarker:  "notmatchingmarker",
-			delimiter:  "/",
-			maxKeys:    "10",
-			statusCode: getAPIError(ErrNotImplemented).HTTPStatusCode,
-		},
-		// 7. Invalid params with unsupported delimiter.
-		{
-			bucket:     "mybucket",
-			prefix:     "prefix",
-			keyMarker:  "notmatchingmarker",
-			delimiter:  "unsupported",
-			maxKeys:    "10",
-			statusCode: getAPIError(ErrNotImplemented).HTTPStatusCode,
-		},
-		// 8. Invalid params with invalid max Keys
-		{
-			bucket:     "mybucket",
-			prefix:     "prefix",
-			keyMarker:  "prefix11",
-			delimiter:  "/",
-			maxKeys:    "999999999999999999999999999",
-			statusCode: getAPIError(ErrInvalidMaxUploads).HTTPStatusCode,
-		},
-	}
-
-	for i, test := range testCases {
-		queryVal := mkListUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys)
-
-		req, err := buildAdminRequest(queryVal, "list-uploads", http.MethodGet, 0, nil)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to construct list uploads needing heal request - %v", i+1, err)
-		}
-
-		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
-
-		if test.statusCode != rec.Code {
-			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
-		}
-
-		// Compare result with the expected one only when we receive 200 OK
-		if rec.Code == http.StatusOK {
-			resp := rec.Result()
-			xmlBytes, err := ioutil.ReadAll(resp.Body)
-			if err != nil {
-				t.Errorf("Test %d: Failed to read response %v", i+1, err)
-			}
-
-			var actualResult ListMultipartUploadsResponse
-			err = xml.Unmarshal(xmlBytes, &actualResult)
-			if err != nil {
-				t.Errorf("Test %d: Failed to unmarshal xml %v", i+1, err)
-			}
-
-			if !reflect.DeepEqual(test.expectedResp, actualResult) {
-				t.Fatalf("Test %d: Unexpected response `%+v`, expected: `%+v`", i+1, test.expectedResp, actualResult)
-			}
-		}
-
-	}
-}
-
 // Test for newHealResult helper function.
 func TestNewHealResult(t *testing.T) {
 	testCases := []struct {

cmd/admin-router.go

@@ -53,8 +53,6 @@ func registerAdminRouter(mux *router.Router) {
 
 	// List Objects needing heal.
 	adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list-objects").HandlerFunc(adminAPI.ListObjectsHealHandler)
-	// List Uploads needing heal.
-	adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list-uploads").HandlerFunc(adminAPI.ListUploadsHealHandler)
 	// List Buckets needing heal.
 	adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list-buckets").HandlerFunc(adminAPI.ListBucketsHealHandler)
 
@@ -64,8 +62,6 @@ func registerAdminRouter(mux *router.Router) {
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler)
 	// Heal Format.
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler)
-	// Heal Uploads.
-	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "upload").HandlerFunc(adminAPI.HealUploadHandler)
 
 	/// Config operations
 

cmd/api-errors.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"encoding/xml"
+	"fmt"
 	"net/http"
 
 	"github.com/minio/minio/pkg/auth"
@@ -175,9 +176,12 @@ const (
 	ErrAdminInvalidAccessKey
 	ErrAdminInvalidSecretKey
 	ErrAdminConfigNoQuorum
+	ErrAdminConfigTooLarge
+	ErrAdminConfigBadJSON
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
+	ErrHealNotImplemented
 )
 
 // error code to APIError structure, these fields carry respective
@@ -712,6 +716,17 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Configuration update failed because server quorum was not met",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrAdminConfigTooLarge: {
+		Code: "XMinioAdminConfigTooLarge",
+		Description: fmt.Sprintf("Configuration data provided exceeds the allowed maximum of %d bytes",
+			maxConfigJSONSize),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminConfigBadJSON: {
+		Code:           "XMinioAdminConfigBadJSON",
+		Description:    "JSON configuration provided has objects with duplicate keys",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminCredentialsMismatch: {
 		Code:           "XMinioAdminCredentialsMismatch",
 		Description:    "Credentials in config mismatch with server environment variables",
@@ -742,6 +757,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    errObjectTampered.Error(),
 		HTTPStatusCode: http.StatusPartialContent,
 	},
+	ErrHealNotImplemented: {
+		Code:           "XMinioHealNotImplemented",
+		Description:    "This server does not implement heal functionality.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrMaximumExpires: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",

cmd/auth-handler.go

@@ -18,16 +18,12 @@ package cmd
 
 import (
 	"bytes"
+	"errors"
 	"io/ioutil"
 	"net/http"
 	"strings"
 )
 
-// Verify if the request http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD"
-func isRequestUnsignedPayload(r *http.Request) bool {
-	return r.Header.Get("x-amz-content-sha256") == unsignedPayload
-}
-
 // Verify if request has JWT.
 func isRequestJWT(r *http.Request) bool {
 	return strings.HasPrefix(r.Header.Get("Authorization"), jwtAlgorithm)
@@ -58,13 +54,14 @@ func isRequestPresignedSignatureV2(r *http.Request) bool {
 
 // Verify if request has AWS Post policy Signature Version '4'.
 func isRequestPostPolicySignatureV4(r *http.Request) bool {
-	return strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") && r.Method == httpPOST
+	return strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") &&
+		r.Method == http.MethodPost
 }
 
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
 func isRequestSignStreamingV4(r *http.Request) bool {
 	return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
-		r.Method == httpPUT
+		r.Method == http.MethodPut
 }
 
 // Authorization type.
@@ -105,6 +102,19 @@ func getRequestAuthType(r *http.Request) authType {
 	return authTypeUnknown
 }
 
+// checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
+// It does not accept presigned or JWT or anonymous requests.
+func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
+	s3Err := ErrAccessDenied
+	if getRequestAuthType(r) == authTypeSigned { // we only support V4 (no presign)
+		s3Err = isReqAuthenticated(r, region)
+	}
+	if s3Err != ErrNone {
+		errorIf(errors.New(getAPIError(s3Err).Description), "%s", dumpRequest(r))
+	}
+	return s3Err
+}
+
 func checkRequestAuthType(r *http.Request, bucket, policyAction, region string) APIErrorCode {
 	reqAuthType := getRequestAuthType(r)
 

cmd/auth-handler_test.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/minio/minio/pkg/auth"
 )
@@ -190,40 +191,6 @@ func TestS3SupportedAuthType(t *testing.T) {
 	}
 }
 
-// TestIsRequestUnsignedPayload - Test validates the Unsigned payload detection logic.
-func TestIsRequestUnsignedPayload(t *testing.T) {
-	testCases := []struct {
-		inputAmzContentHeader string
-		expectedResult        bool
-	}{
-		// Test case - 1.
-		// Test case with "X-Amz-Content-Sha256" header set to empty value.
-		{"", false},
-		// Test case - 2.
-		// Test case with "X-Amz-Content-Sha256" header set to  "UNSIGNED-PAYLOAD"
-		// The payload is flagged as unsigned When "X-Amz-Content-Sha256" header is set to  "UNSIGNED-PAYLOAD".
-		{unsignedPayload, true},
-		// Test case - 3.
-		// set to a random value.
-		{"abcd", false},
-	}
-
-	// creating an input HTTP request.
-	// Only the headers are relevant for this particular test.
-	inputReq, err := http.NewRequest("GET", "http://example.com", nil)
-	if err != nil {
-		t.Fatalf("Error initializing input HTTP request: %v", err)
-	}
-
-	for i, testCase := range testCases {
-		inputReq.Header.Set("X-Amz-Content-Sha256", testCase.inputAmzContentHeader)
-		actualResult := isRequestUnsignedPayload(inputReq)
-		if testCase.expectedResult != actualResult {
-			t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult)
-		}
-	}
-}
-
 func TestIsRequestPresignedSignatureV2(t *testing.T) {
 	testCases := []struct {
 		inputQueryKey   string
@@ -311,6 +278,39 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod
 	return req
 }
 
+// This is similar to mustNewRequest but additionally the request
+// is signed with AWS Signature V2, fails if not able to do so.
+func mustNewSignedV2Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
+	req := mustNewRequest(method, urlStr, contentLength, body, t)
+	cred := globalServerConfig.GetCredential()
+	if err := signRequestV2(req, cred.AccessKey, cred.SecretKey); err != nil {
+		t.Fatalf("Unable to inititalized new signed http request %s", err)
+	}
+	return req
+}
+
+// This is similar to mustNewRequest but additionally the request
+// is presigned with AWS Signature V2, fails if not able to do so.
+func mustNewPresignedV2Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
+	req := mustNewRequest(method, urlStr, contentLength, body, t)
+	cred := globalServerConfig.GetCredential()
+	if err := preSignV2(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
+		t.Fatalf("Unable to inititalized new signed http request %s", err)
+	}
+	return req
+}
+
+// This is similar to mustNewRequest but additionally the request
+// is presigned with AWS Signature V4, fails if not able to do so.
+func mustNewPresignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
+	req := mustNewRequest(method, urlStr, contentLength, body, t)
+	cred := globalServerConfig.GetCredential()
+	if err := preSignV4(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
+		t.Fatalf("Unable to inititalized new signed http request %s", err)
+	}
+	return req
+}
+
 func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	req.Header.Set("Content-Md5", "YWFhYWFhYWFhYWFhYWFhCg==")
@@ -358,3 +358,32 @@ func TestIsReqAuthenticated(t *testing.T) {
 		}
 	}
 }
+func TestCheckAdminRequestAuthType(t *testing.T) {
+	path, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer os.RemoveAll(path)
+
+	creds, err := auth.CreateCredentials("myuser", "mypassword")
+	if err != nil {
+		t.Fatalf("unable create credential, %s", err)
+	}
+
+	globalServerConfig.SetCredential(creds)
+	testCases := []struct {
+		Request *http.Request
+		ErrCode APIErrorCode
+	}{
+		{Request: mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
+		{Request: mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrNone},
+		{Request: mustNewSignedV2Request("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
+		{Request: mustNewPresignedV2Request("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
+		{Request: mustNewPresignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
+	}
+	for i, testCase := range testCases {
+		if s3Error := checkAdminRequestAuthType(testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
+			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
+		}
+	}
+}

cmd/benchmark-utils_test.go

@@ -239,7 +239,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		var buffer = new(bytes.Buffer)
-		err = obj.GetObject(bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer)
+		err = obj.GetObject(bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "")
 		if err != nil {
 			b.Error(err)
 		}
@@ -402,7 +402,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	b.RunParallel(func(pb *testing.PB) {
 		i := 0
 		for pb.Next() {
-			err = obj.GetObject(bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard)
+			err = obj.GetObject(bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "")
 			if err != nil {
 				b.Error(err)
 			}

cmd/bucket-handlers-listobjects.go

@@ -115,7 +115,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	// marshalled into S3 compatible XML header.
 	listObjectsInfo, err := objectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		errorIf(err, "Unable to list objects.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -161,7 +160,6 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	// marshalled into S3 compatible XML header.
 	listObjectsInfo, err := objectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		errorIf(err, "Unable to list objects.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}

cmd/bucket-handlers.go

@@ -128,7 +128,6 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 	}
 
 	if _, err := objectAPI.GetBucketInfo(bucket); err != nil {
-		errorIf(err, "Unable to fetch bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -185,7 +184,6 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 
 	listMultipartsInfo, err := objectAPI.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
-		errorIf(err, "Unable to list multipart uploads.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -221,7 +219,6 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	// Invoke the list buckets.
 	bucketsInfo, err := objectAPI.ListBuckets()
 	if err != nil {
-		errorIf(err, "Unable to list buckets.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -304,16 +301,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 				}
 				return
 			}
-			objectLock := globalNSMutex.NewNSLock(bucket, obj.ObjectName)
-			if timedOutErr := objectLock.GetLock(globalObjectTimeout); timedOutErr != nil {
-				dErrs[i] = timedOutErr
-			} else {
-				defer objectLock.Unlock()
-
-				dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
-				if dErr != nil {
-					dErrs[i] = dErr
-				}
+			dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
+			if dErr != nil {
+				dErrs[i] = dErr
 			}
 		}(index, object)
 	}
@@ -335,7 +325,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			deletedObjects = append(deletedObjects, object)
 			continue
 		}
-		errorIf(err, "Unable to delete object. %s", object.ObjectName)
 		// Error during delete should be collected separately.
 		deleteErrors = append(deleteErrors, DeleteError{
 			Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
@@ -408,17 +397,9 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if bucketLock.GetLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
-	defer bucketLock.Unlock()
-
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(bucket, "")
 	if err != nil {
-		errorIf(err, "Unable to create a bucket.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -546,13 +527,11 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	lengthRange := postPolicyForm.Conditions.ContentLengthRange
 	if lengthRange.Valid {
 		if fileSize < lengthRange.Min {
-			errorIf(err, "Unable to create object.")
 			writeErrorResponse(w, toAPIErrorCode(errDataTooSmall), r.URL)
 			return
 		}
 
 		if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
-			errorIf(err, "Unable to create object.")
 			writeErrorResponse(w, toAPIErrorCode(errDataTooLarge), r.URL)
 			return
 		}
@@ -566,13 +545,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	objectLock := globalNSMutex.NewNSLock(bucket, object)
-	if objectLock.GetLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
-	defer objectLock.Unlock()
-
 	hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
 	if err != nil {
 		errorIf(err, "Unable to initialize hashReader.")
@@ -582,7 +554,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	objInfo, err := objectAPI.PutObject(bucket, object, hashReader, metadata)
 	if err != nil {
-		errorIf(err, "Unable to create object.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -662,7 +633,6 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 	defer bucketLock.RUnlock()
 
 	if _, err := objectAPI.GetBucketInfo(bucket); err != nil {
-		errorIf(err, "Unable to fetch bucket info.")
 		writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
 		return
 	}
@@ -696,7 +666,6 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	// Attempt to delete bucket.
 	if err := objectAPI.DeleteBucket(bucket); err != nil {
-		errorIf(err, "Unable to delete a bucket.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}

cmd/bucket-notification-handlers.go

@@ -59,7 +59,6 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	_, err := objAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -113,7 +112,6 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	_, err := objectAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -328,7 +326,6 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	_, err := objAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to get bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}

cmd/bucket-policy-handlers.go

@@ -236,7 +236,6 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -295,7 +294,6 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -336,7 +334,6 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -344,7 +341,6 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	// Read bucket access policy.
 	policy, err := readBucketPolicy(bucket, objAPI)
 	if err != nil {
-		errorIf(err, "Unable to read bucket policy.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}

cmd/bucket-policy.go

@@ -88,7 +88,6 @@ func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]policy.Bucke
 	// List buckets to proceed loading all notification configuration.
 	buckets, err := objAPI.ListBuckets()
 	if err != nil {
-		errorIf(err, "Unable to list buckets.")
 		return nil, errors.Cause(err)
 	}
 
@@ -149,15 +148,8 @@ func initBucketPolicies(objAPI ObjectLayer) error {
 func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader io.Reader, err error) {
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
 
-	// Acquire a read lock on policy config before reading.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	if err = objLock.GetRLock(globalOperationTimeout); err != nil {
-		return nil, err
-	}
-	defer objLock.RUnlock()
-
 	var buffer bytes.Buffer
-	err = objAPI.GetObject(minioMetaBucket, policyPath, 0, -1, &buffer)
+	err = objAPI.GetObject(minioMetaBucket, policyPath, 0, -1, &buffer, "")
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
 			return nil, BucketPolicyNotFound{Bucket: bucket}
@@ -191,15 +183,8 @@ func readBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPol
 // if no policies are found.
 func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-	// Acquire a write lock on policy config before modifying.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
 	err := objAPI.DeleteObject(minioMetaBucket, policyPath)
 	if err != nil {
-		errorIf(err, "Unable to remove bucket-policy on bucket %s.", bucket)
 		err = errors.Cause(err)
 		if _, ok := err.(ObjectNotFound); ok {
 			return BucketPolicyNotFound{Bucket: bucket}
@@ -217,13 +202,6 @@ func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy policy.BucketAcces
 		return err
 	}
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-	// Acquire a write lock on policy config before modifying.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	if err = objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
-
 	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
 	if err != nil {
 		errorIf(err, "Unable to set policy for the bucket %s", bucket)
@@ -242,7 +220,6 @@ func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI Objec
 	var bktPolicy policy.BucketAccessPolicy
 	err := parseBucketPolicy(bytes.NewReader(policyBytes), &bktPolicy)
 	if err != nil {
-		errorIf(err, "Unable to parse bucket policy.")
 		return ErrInvalidPolicyDocument
 	}
 

cmd/certs.go

@@ -22,9 +22,15 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 )
 
+// TLSPrivateKeyPassword is the environment variable which contains the password used
+// to decrypt the TLS private key. It must be set if the TLS private key is
+// password protected.
+const TLSPrivateKeyPassword = "MINIO_CERT_PASSWD"
+
 func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
 	// Read certificate file.
 	var data []byte
@@ -90,6 +96,36 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	return rootCAs, nil
 }
 
+// load an X509 key pair (private key , certificate) from the provided
+// paths. The private key may be encrypted and is decrypted using the
+// ENV_VAR: MINIO_CERT_PASSWD.
+func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
+	certPEMBlock, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return tls.Certificate{}, fmt.Errorf("TLS: failed to read cert file: %v", err)
+	}
+	keyPEMBlock, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		return tls.Certificate{}, fmt.Errorf("TLS: failed to read private key: %v", err)
+	}
+	key, rest := pem.Decode(keyPEMBlock)
+	if len(rest) > 0 {
+		return tls.Certificate{}, fmt.Errorf("TLS: private key contains additional data")
+	}
+	if x509.IsEncryptedPEMBlock(key) {
+		password, ok := os.LookupEnv(TLSPrivateKeyPassword)
+		if !ok {
+			return tls.Certificate{}, fmt.Errorf("TLS: private key is encrypted but no password is present - set env var: %s", TLSPrivateKeyPassword)
+		}
+		decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
+		if decErr != nil {
+			return tls.Certificate{}, fmt.Errorf("TLS: failed to decrypt private key: %v", decErr)
+		}
+		keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
+	}
+	return tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+}
+
 func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
 		return nil, nil, nil, false, nil
@@ -100,7 +136,7 @@ func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsC
 	}
 
 	var cert tls.Certificate
-	if cert, err = tls.LoadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
+	if cert, err = loadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
 		return nil, nil, nil, false, err
 	}
 

cmd/certs_test.go

@@ -257,3 +257,252 @@ func TestGetRootCAs(t *testing.T) {
 		}
 	}
 }
+
+func TestLoadX509KeyPair(t *testing.T) {
+	for i, testCase := range loadX509KeyPairTests {
+		privateKey, err := createTempFile("private.key", testCase.privateKey)
+		if err != nil {
+			t.Fatalf("Test %d: failed to create tmp private key file: %v", i, err)
+		}
+		certificate, err := createTempFile("public.crt", testCase.certificate)
+		if err != nil {
+			os.Remove(privateKey)
+			t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err)
+		}
+
+		os.Unsetenv(TLSPrivateKeyPassword)
+		if testCase.password != "" {
+			os.Setenv(TLSPrivateKeyPassword, testCase.password)
+		}
+		_, err = loadX509KeyPair(certificate, privateKey)
+		if err != nil && !testCase.shouldFail {
+			t.Errorf("Test %d: test should succeed but it failed: %v", i, err)
+		}
+		if err == nil && testCase.shouldFail {
+			t.Errorf("Test %d: test should fail but it succeed", i)
+		}
+		os.Remove(privateKey)
+		os.Remove(certificate)
+	}
+}
+
+var loadX509KeyPairTests = []struct {
+	password                string
+	privateKey, certificate string
+	shouldFail              bool
+}{
+	{
+		password: "foobar",
+		privateKey: `-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-128-CBC,CC483BF11678C35F9F02A1AD85DAE285
+
+nMDFd+Qxk1f+S7LwMitmMofNXYNbCY4L1QEqPOOx5wnjNF1wSxmEkL7+h8W4Y/vb
+AQt/7TCcUSuSqEMl45nUIcCbhBos5wz+ShvFiez3qKwmR5HSURvqyN6PIJeAbU+h
+uw/cvAQsCH1Cq+gYkDJqjrizPhGqg7mSkqyeST3PbOl+ZXc0wynIjA34JSwO3c5j
+cF7XKHETtNGj1+AiLruX4wYZAJwQnK375fCoNVMO992zC6K83d8kvGMUgmJjkiIj
+q3s4ymFGfoo0S/XNDQXgE5A5QjAKRKUyW2i7pHIIhTyOpeJQeFHDi2/zaZRxoCog
+lD2/HKLi5xJtRelZaaGyEJ20c05VzaSZ+EtRIN33foNdyQQL6iAUU3hJ6JlcmRIB
+bRfX4XPH1w9UfFU5ZKwUciCoDcL65bsyv/y56ItljBp7Ok+UUKl0H4myFNOSfsuU
+IIj4neslnAvwQ8SN4XUpug+7pGF+2m/5UDwRzSUN1H2RfgWN95kqR+tYqCq/E+KO
+i0svzFrljSHswsFoPBqKngI7hHwc9QTt5q4frXwj9I4F6HHrTKZnC5M4ef26sbJ1
+r7JRmkt0h/GfcS355b0uoBTtF1R8tSJo85Zh47wE+ucdjEvy9/pjnzKqIoJo9bNZ
+ri+ue7GhH5EUca1Kd10bH8FqTF+8AHh4yW6xMxSkSgFGp7KtraAVpdp+6kosymqh
+dz9VMjA8i28btfkS2isRaCpyumaFYJ3DJMFYhmeyt6gqYovmRLX0qrBf8nrkFTAA
+ZmykWsc8ErsCudxlDmKVemuyFL7jtm9IRPq+Jh+IrmixLJFx8PKkNAM6g+A8irx8
+piw+yhRsVy5Jk2QeIqvbpxN6BfCNcix4sWkusiCJrAqQFuSm26Mhh53Ig1DXG4d3
+6QY1T8tW80Q6JHUtDR+iOPqW6EmrNiEopzirvhGv9FicXZ0Lo2yKJueeeihWhFLL
+GmlnCjWVMO4hoo8lWCHv95JkPxGMcecCacKKUbHlXzCGyw3+eeTEHMWMEhziLeBy
+HZJ1/GReI3Sx7XlUCkG4468Yz3PpmbNIk/U5XKE7TGuxKmfcWQpu022iF/9DrKTz
+KVhKimCBXJX345bCFe1rN2z5CV6sv87FkMs5Y+OjPw6qYFZPVKO2TdUUBcpXbQMg
+UW+Kuaax9W7214Stlil727MjRCiH1+0yODg4nWj4pTSocA5R3pn5cwqrjMu97OmL
+ESx4DHmy4keeSy3+AIAehCZlwgeLb70/xCSRhJMIMS9Q6bz8CPkEWN8bBZt95oeo
+37LqZ7lNmq61fs1x1tq0VUnI9HwLFEnsiubp6RG0Yu8l/uImjjjXa/ytW2GXrfUi
+zM22dOntu6u23iBxRBJRWdFTVUz7qrdu+PHavr+Y7TbCeiBwiypmz5llf823UIVx
+btamI6ziAq2gKZhObIhut7sjaLkAyTLlNVkNN1WNaplAXpW25UFVk93MHbvZ27bx
+9iLGs/qB2kDTUjffSQoHTLY1GoLxv83RgVspUGQjslztEEpWfYvGfVLcgYLv933B
+aRW9BRoNZ0czKx7Lhuwjreyb5IcWDarhC8q29ZkkWsQQonaPb0kTEFJul80Yqk0k
+-----END RSA PRIVATE KEY-----`,
+		certificate: `-----BEGIN CERTIFICATE-----
+MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
+BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
+BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
+MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
+YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
+EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
+AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6
+HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR
+GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc
+VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla
+Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd
+cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD
+0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV
+HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy
+iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ
+b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj
+KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb
+bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl
+kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR
+-----END CERTIFICATE-----`,
+		shouldFail: false,
+	},
+	{
+		password: "password",
+		privateKey: `-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-128-CBC,CC483BF11678C35F9F02A1AD85DAE285
+
+nMDFd+Qxk1f+S7LwMitmMofNXYNbCY4L1QEqPOOx5wnjNF1wSxmEkL7+h8W4Y/vb
+AQt/7TCcUSuSqEMl45nUIcCbhBos5wz+ShvFiez3qKwmR5HSURvqyN6PIJeAbU+h
+uw/cvAQsCH1Cq+gYkDJqjrizPhGqg7mSkqyeST3PbOl+ZXc0wynIjA34JSwO3c5j
+cF7XKHETtNGj1+AiLruX4wYZAJwQnK375fCoNVMO992zC6K83d8kvGMUgmJjkiIj
+q3s4ymFGfoo0S/XNDQXgE5A5QjAKRKUyW2i7pHIIhTyOpeJQeFHDi2/zaZRxoCog
+lD2/HKLi5xJtRelZaaGyEJ20c05VzaSZ+EtRIN33foNdyQQL6iAUU3hJ6JlcmRIB
+bRfX4XPH1w9UfFU5ZKwUciCoDcL65bsyv/y56ItljBp7Ok+UUKl0H4myFNOSfsuU
+IIj4neslnAvwQ8SN4XUpug+7pGF+2m/5UDwRzSUN1H2RfgWN95kqR+tYqCq/E+KO
+i0svzFrljSHswsFoPBqKngI7hHwc9QTt5q4frXwj9I4F6HHrTKZnC5M4ef26sbJ1
+r7JRmkt0h/GfcS355b0uoBTtF1R8tSJo85Zh47wE+ucdjEvy9/pjnzKqIoJo9bNZ
+ri+ue7GhH5EUca1Kd10bH8FqTF+8AHh4yW6xMxSkSgFGp7KtraAVpdp+6kosymqh
+dz9VMjA8i28btfkS2isRaCpyumaFYJ3DJMFYhmeyt6gqYovmRLX0qrBf8nrkFTAA
+ZmykWsc8ErsCudxlDmKVemuyFL7jtm9IRPq+Jh+IrmixLJFx8PKkNAM6g+A8irx8
+piw+yhRsVy5Jk2QeIqvbpxN6BfCNcix4sWkusiCJrAqQFuSm26Mhh53Ig1DXG4d3
+6QY1T8tW80Q6JHUtDR+iOPqW6EmrNiEopzirvhGv9FicXZ0Lo2yKJueeeihWhFLL
+GmlnCjWVMO4hoo8lWCHv95JkPxGMcecCacKKUbHlXzCGyw3+eeTEHMWMEhziLeBy
+HZJ1/GReI3Sx7XlUCkG4468Yz3PpmbNIk/U5XKE7TGuxKmfcWQpu022iF/9DrKTz
+KVhKimCBXJX345bCFe1rN2z5CV6sv87FkMs5Y+OjPw6qYFZPVKO2TdUUBcpXbQMg
+UW+Kuaax9W7214Stlil727MjRCiH1+0yODg4nWj4pTSocA5R3pn5cwqrjMu97OmL
+ESx4DHmy4keeSy3+AIAehCZlwgeLb70/xCSRhJMIMS9Q6bz8CPkEWN8bBZt95oeo
+37LqZ7lNmq61fs1x1tq0VUnI9HwLFEnsiubp6RG0Yu8l/uImjjjXa/ytW2GXrfUi
+zM22dOntu6u23iBxRBJRWdFTVUz7qrdu+PHavr+Y7TbCeiBwiypmz5llf823UIVx
+btamI6ziAq2gKZhObIhut7sjaLkAyTLlNVkNN1WNaplAXpW25UFVk93MHbvZ27bx
+9iLGs/qB2kDTUjffSQoHTLY1GoLxv83RgVspUGQjslztEEpWfYvGfVLcgYLv933B
+aRW9BRoNZ0czKx7Lhuwjreyb5IcWDarhC8q29ZkkWsQQonaPb0kTEFJul80Yqk0k
+-----END RSA PRIVATE KEY-----`,
+		certificate: `-----BEGIN CERTIFICATE-----
+MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
+BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
+BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
+MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
+YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
+EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
+AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6
+HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR
+GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc
+VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla
+Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd
+cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD
+0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV
+HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy
+iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ
+b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj
+KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb
+bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl
+kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR
+-----END CERTIFICATE-----`,
+		shouldFail: true,
+	},
+	{
+		password: "",
+		privateKey: `-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA4K9Qq7vMY2bGkrdFAYpBYNLlCgnnFU+0pi+N+3bjuWmfX/kw
+WXBa3SDqKD08PWWzwvBSLPCCUV2IuUd7tBa1pJ2wXkdoDeI5InYHJKrXbSZonni6
+Bex7sgnqV/9o8xFkSOleoQWZgyeKGxtt0J/Z+zhpH+zXahwM4wOL3yzLSQt+NCKM
+6N96zXYi16DEa89fYwRxPwE1XTRc7Ddggqx+4iRHvYG0fyTNcPB/+UiFw59EE1Sg
+QIyTVntVqpsb6s8XdkFxURoLxefhcMVf2kU0T04OWI3gmeavKfTcj8Z2/bjPSsqP
+mgkADv9Ru6VnSK/96TW/NwxWJ32PBz6Sbl9LdwIDAQABAoIBABVh+d5uH/RxyoIZ
++PI9kx1A1NVQvfI0RK/wJKYC2YdCuw0qLOTGIY+b20z7DumU7TenIVrvhKdzrFhd
+qjMoWh8RdsByMT/pAKD79JATxi64EgrK2IFJ0TfPY8L+JqHDTPT3aK8QVly5/ZW4
+1YmePOOAqdiE9Lc/diaApuYVYD9SL/X7fYs1ezOB4oGXoz0rthX77zHMxcEurpK3
+VgSnaq7FYTVY7GrFB+ASiAlDIyLwztz08Ijn8aG0QAZ8GFuPGSmPMXWjLwFhRZsa
+Gfy5BYiA0bVSnQSPHzAnHu9HyGlsdouVPPvJB3SrvMl+BFhZiUuR8OGSob7z7hfI
+hMyHbNECgYEA/gyG7sHAb5mPkhq9JkTv+LrMY5NDZKYcSlbvBlM3kd6Ib3Hxl+6T
+FMq2TWIrh2+mT1C14htziHd05dF6St995Tby6CJxTj6a/2Odnfm+JcOou/ula4Sz
+92nIGlGPTJXstDbHGnRCpk6AomXK02stydTyrCisOw1H+LyTG6aT0q8CgYEA4mkO
+hfLJkgmJzWIhxHR901uWHz/LId0gC6FQCeaqWmRup6Bl97f0U6xokw4tw8DJOncF
+yZpYRXUXhdv/FXCjtXvAhKIX5+e+3dlzPHIdekSfcY00ip/ifAS1OyVviJia+cna
+eJgq8WLHxJZim9Ah93NlPyiqGPwtasub90qjZbkCgYEA35WK02o1wII3dvCNc7bM
+M+3CoAglEdmXoF1uM/TdPUXKcbqoU3ymeXAGjYhOov3CMp/n0z0xqvLnMLPxmx+i
+ny6DDYXyjlhO9WFogHYhwP636+mHJl8+PAsfDvqk0VRJZDmpdUDIv7DrSQGpRfRX
+8f+2K4oIOlhv9RuRpI4wHwUCgYB8OjaMyn1NEsy4k2qBt4U+jhcdyEv1pbWqi/U1
+qYm5FTgd44VvWVDHBGdQoMv9h28iFCJpzrU2Txv8B4y7v9Ujg+ZLIAFL7j0szt5K
+wTZpWvO9Q0Qb98Q2VgL2lADRiyIlglrMJnoRfiisNfOfGKE6e+eGsxI5qUxmN5e5
+JQvoiQKBgQCqgyuUBIu/Qsb3qUED/o0S5wCel43Yh/Rl+mxDinOUvJfKJSW2SyEk
++jDo0xw3Opg6ZC5Lj2V809LA/XteaIuyhRuqOopjhHIvIvrYGe+2O8q9/Mv40BYW
+0BhJ/Gdseps0C6Z5mTT5Fee4YVlGZuyuNKmKTd4JmqInfBV3ncMWQg==
+-----END RSA PRIVATE KEY-----`,
+		certificate: `-----BEGIN CERTIFICATE-----
+MIIDiTCCAnGgAwIBAgIJAIb84Z5Mh31iMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
+BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
+BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
+NTcyM1oXDTI3MTIxNjE4NTcyM1owWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
+YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
+EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
+AQDgr1Cru8xjZsaSt0UBikFg0uUKCecVT7SmL437duO5aZ9f+TBZcFrdIOooPTw9
+ZbPC8FIs8IJRXYi5R3u0FrWknbBeR2gN4jkidgckqtdtJmieeLoF7HuyCepX/2jz
+EWRI6V6hBZmDJ4obG23Qn9n7OGkf7NdqHAzjA4vfLMtJC340Iozo33rNdiLXoMRr
+z19jBHE/ATVdNFzsN2CCrH7iJEe9gbR/JM1w8H/5SIXDn0QTVKBAjJNWe1Wqmxvq
+zxd2QXFRGgvF5+FwxV/aRTRPTg5YjeCZ5q8p9NyPxnb9uM9Kyo+aCQAO/1G7pWdI
+r/3pNb83DFYnfY8HPpJuX0t3AgMBAAGjUDBOMB0GA1UdDgQWBBQ2/bSCHscnoV+0
+d+YJxLu4XLSNIDAfBgNVHSMEGDAWgBQ2/bSCHscnoV+0d+YJxLu4XLSNIDAMBgNV
+HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC6p4gPwmkoDtRsP1c8IWgXFka+
+Q59oe79ZK1RqDE6ZZu0rgw07rPzKr4ofW4hTxnx7PUgKOhWLq9VvwEC/9tDbD0Gw
+SKknRZZOiEE3qUZbwNtHMd4UBzpzChTRC6RcwC5zT1/WICMUHxa4b8E2umJuf3Qd
+5Y23sXEESx5evr49z6DLcVe2i70o2wJeWs2kaXqhCJt0X7z0rnYqjfFdvxd8dyzt
+1DXmE45cLadpWHDg26DMsdchamgnqEo79YUxkH6G/Cb8ZX4igQ/CsxCDOKvccjHO
+OncDtuIpK8O7OyfHP3+MBpUFG4P6Ctn7RVcZe9fQweTpfAy18G+loVzuUeOD
+-----END CERTIFICATE-----`,
+		shouldFail: false,
+	},
+	{
+		password: "foobar",
+		privateKey: `-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA4K9Qq7vMY2bGkrdFAYpBYNLlCgnnFU+0pi+N+3bjuWmfX/kw
+WXBa3SDqKD08PWWzwvBSLPCCUV2IuUd7tBa1pJ2wXkdoDeI5InYHJKrXbSZonni6
+Bex7sgnqV/9o8xFkSOleoQWZgyeKGxtt0J/Z+zhpH+zXahwM4wOL3yzLSQt+NCKM
+6N96zXYi16DEa89fYwRxPwE1XTRc7Ddggqx+4iRHvYG0fyTNcPB/+UiFw59EE1Sg
+QIyTVntVqpsb6s8XdkFxURoLxefhcMVf2kU0T04OWI3gmeavKfTcj8Z2/bjPSsqP
+mgkADv9Ru6VnSK/96TW/NwxWJ32PBz6Sbl9LdwIDAQABAoIBABVh+d5uH/RxyoIZ
++PI9kx1A1NVQvfI0RK/wJKYC2YdCuw0qLOTGIY+b20z7DumU7TenIVrvhKdzrFhd
+qjMoWh8RdsByMT/pAKD79JATxi64EgrK2IFJ0TfPY8L+JqHDTPT3aK8QVly5/ZW4
+1YmePOOAqdiE9Lc/diaApuYVYD9SL/X7fYs1ezOB4oGXoz0rthX77zHMxcEurpK3
+VgSnaq7FYTVY7GrFB+ASiAlDIyLwztz08Ijn8aG0QAZ8GFuPGSmPMXWjLwFhRZsa
+Gfy5BYiA0bVSnQSPHzAnHu9HyGlsdouVPPvJB3SrvMl+BFhZiUuR8OGSob7z7hfI
+hMyHbNECgYEA/gyG7sHAb5mPkhq9JkTv+LrMY5NDZKYcSlbvBlM3kd6Ib3Hxl+6T
+FMq2TWIrh2+mT1C14htziHd05dF6St995Tby6CJxTj6a/2Odnfm+JcOou/ula4Sz
+92nIGlGPTJXstDbHGnRCpk6AomXK02stydTyrCisOw1H+LyTG6aT0q8CgYEA4mkO
+hfLJkgmJzWIhxHR901uWHz/LId0gC6FQCeaqWmRup6Bl97f0U6xokw4tw8DJOncF
+yZpYRXUXhdv/FXCjtXvAhKIX5+e+3dlzPHIdekSfcY00ip/ifAS1OyVviJia+cna
+eJgq8WLHxJZim9Ah93NlPyiqGPwtasub90qjZbkCgYEA35WK02o1wII3dvCNc7bM
+M+3CoAglEdmXoF1uM/TdPUXKcbqoU3ymeXAGjYhOov3CMp/n0z0xqvLnMLPxmx+i
+ny6DDYXyjlhO9WFogHYhwP636+mHJl8+PAsfDvqk0VRJZDmpdUDIv7DrSQGpRfRX
+8f+2K4oIOlhv9RuRpI4wHwUCgYB8OjaMyn1NEsy4k2qBt4U+jhcdyEv1pbWqi/U1
+qYm5FTgd44VvWVDHBGdQoMv9h28iFCJpzrU2Txv8B4y7v9Ujg+ZLIAFL7j0szt5K
+wTZpWvO9Q0Qb98Q2VgL2lADRiyIlglrMJnoRfiisNfOfGKE6e+eGsxI5qUxmN5e5
+JQvoiQKBgQCqgyuUBIu/Qsb3qUED/o0S5wCel43Yh/Rl+mxDinOUvJfKJSW2SyEk
++jDo0xw3Opg6ZC5Lj2V809LA/XteaIuyhRuqOopjhHIvIvrYGe+2O8q9/Mv40BYW
+0BhJ/Gdseps0C6Z5mTT5Fee4YVlGZuyuNKmKTd4JmqInfBV3ncMWQg==
+-----END RSA PRIVATE KEY-----`,
+		certificate: `-----BEGIN CERTIFICATE-----
+MIIDiTCCAnGgAwIBAgIJAIb84Z5Mh31iMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
+BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
+BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
+NTcyM1oXDTI3MTIxNjE4NTcyM1owWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
+YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
+EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
+AQDgr1Cru8xjZsaSt0UBikFg0uUKCecVT7SmL437duO5aZ9f+TBZcFrdIOooPTw9
+ZbPC8FIs8IJRXYi5R3u0FrWknbBeR2gN4jkidgckqtdtJmieeLoF7HuyCepX/2jz
+EWRI6V6hBZmDJ4obG23Qn9n7OGkf7NdqHAzjA4vfLMtJC340Iozo33rNdiLXoMRr
+z19jBHE/ATVdNFzsN2CCrH7iJEe9gbR/JM1w8H/5SIXDn0QTVKBAjJNWe1Wqmxvq
+zxd2QXFRGgvF5+FwxV/aRTRPTg5YjeCZ5q8p9NyPxnb9uM9Kyo+aCQAO/1G7pWdI
+r/3pNb83DFYnfY8HPpJuX0t3AgMBAAGjUDBOMB0GA1UdDgQWBBQ2/bSCHscnoV+0
+d+YJxLu4XLSNIDAfBgNVHSMEGDAWgBQ2/bSCHscnoV+0d+YJxLu4XLSNIDAMBgNV
+HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC6p4gPwmkoDtRsP1c8IWgXFka+
+Q59oe79ZK1RqDE6ZZu0rgw07rPzKr4ofW4hTxnx7PUgKOhWLq9VvwEC/9tDbD0Gw
+SKknRZZOiEE3qUZbwNtHMd4UBzpzChTRC6RcwC5zT1/WICMUHxa4b8E2umJuf3Qd
+5Y23sXEESx5evr49z6DLcVe2i70o2wJeWs2kaXqhCJt0X7z0rnYqjfFdvxd8dyzt
+1DXmE45cLadpWHDg26DMsdchamgnqEo79YUxkH6G/Cb8ZX4igQ/CsxCDOKvccjHO
+OncDtuIpK8O7OyfHP3+MBpUFG4P6Ctn7RVcZe9fQweTpfAy18G+loVzuUeOD
+-----END CERTIFICATE-----`,
+		shouldFail: false,
+	},
+}

cmd/common-main.go

@@ -102,7 +102,12 @@ func handleCommonEnvVars() {
 		globalIsBrowserEnabled = bool(browserFlag)
 	}
 
-	globalHTTPTrace = os.Getenv("MINIO_HTTP_TRACE") != ""
+	traceFile := os.Getenv("MINIO_HTTP_TRACE")
+	if traceFile != "" {
+		var err error
+		globalHTTPTraceFile, err = os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
+		fatalIf(err, "error opening file %s", traceFile)
+	}
 
 	globalDomainName = os.Getenv("MINIO_DOMAIN")
 	if globalDomainName != "" {

cmd/config-current.go

@@ -20,7 +20,6 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"strconv"
 	"sync"
 
 	"github.com/minio/minio/pkg/auth"
@@ -108,46 +107,35 @@ func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
 	s.Lock()
 	defer s.Unlock()
 
-	// Set the values
-	s.StorageClass.Standard = standardClass.Scheme + strconv.Itoa(standardClass.Parity)
-	s.StorageClass.RRS = rrsClass.Scheme + strconv.Itoa(rrsClass.Parity)
+	s.StorageClass.Standard = standardClass
+	s.StorageClass.RRS = rrsClass
 }
 
-func (s *serverConfig) GetStorageClass() (standardStorageClass, rrsStorageClass storageClass) {
+// GetStorageClass reads storage class fields from current config, parses and validates it.
+// It returns the standard and reduced redundancy storage class struct
+func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
 	s.RLock()
 	defer s.RUnlock()
 
 	var err error
-	var ssc storageClass
-	var rrsc storageClass
+	// Storage Class from config.json is already parsed and stored in s.StorageClass
+	// Now validate the storage class fields
+	ssc := s.StorageClass.Standard
+	rrsc := s.StorageClass.RRS
 
-	if s.StorageClass.Standard != "" {
-		// Parse the values read from config file into storageClass struct
-		ssc, err = parseStorageClass(s.StorageClass.Standard)
-		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.Standard)
-	}
-
-	if s.StorageClass.RRS != "" {
-		// Parse the values read from config file into storageClass struct
-		rrsc, err = parseStorageClass(s.StorageClass.RRS)
-		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.RRS)
-	}
-
-	// Validation is done after parsing both the storage classes. This is needed because we need one
-	// storage class value to deduce the correct value of the other storage class.
 	if rrsc.Scheme != "" {
-		err := validateRRSParity(rrsc.Parity, ssc.Parity)
-		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.RRS)
+		err = validateRRSParity(rrsc.Parity, ssc.Parity)
+		fatalIf(err, "Invalid value %s:%d set in config.json", rrsc.Scheme, rrsc.Parity)
 		globalIsStorageClass = true
 	}
 
 	if ssc.Scheme != "" {
-		err := validateSSParity(ssc.Parity, rrsc.Parity)
-		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.Standard)
+		err = validateSSParity(ssc.Parity, rrsc.Parity)
+		fatalIf(err, "Invalid value %s:%d set in config.json", ssc.Scheme, ssc.Parity)
 		globalIsStorageClass = true
 	}
 
-	return
+	return s.StorageClass.Standard, s.StorageClass.RRS
 }
 
 // GetCredentials get current credentials.
@@ -169,11 +157,12 @@ func (s *serverConfig) Save() error {
 
 func newServerConfig() *serverConfig {
 	srvCfg := &serverConfig{
-		Version:    serverConfigVersion,
-		Credential: auth.MustGetNewCredentials(),
-		Region:     globalMinioDefaultRegion,
-		Browser:    true,
-		Notify:     &notifier{},
+		Version:      serverConfigVersion,
+		Credential:   auth.MustGetNewCredentials(),
+		Region:       globalMinioDefaultRegion,
+		Browser:      true,
+		StorageClass: storageClassConfig{},
+		Notify:       &notifier{},
 	}
 
 	// Make sure to initialize notification configs.

cmd/config-migrate.go

@@ -1147,14 +1147,6 @@ func migrateV15ToV16() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv15.Browser
 
-	// Migrate console and file fields
-	if cv15.Logger.Console.Enable {
-		srvConfig.Logger.Console = NewConsoleLogger()
-	}
-	if cv15.Logger.File.Enable {
-		srvConfig.Logger.File = NewFileLogger(cv15.Logger.File.Filename)
-	}
-
 	if err = quick.Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err)
 	}

cmd/config-versions.go

@@ -418,6 +418,21 @@ type serverConfigV15 struct {
 	Notify *notifier `json:"notify"`
 }
 
+// FileLogger is introduced to workaround the dependency about logrus
+type FileLogger struct {
+	Enable   bool   `json:"enable"`
+	Filename string `json:"filename"`
+}
+
+// ConsoleLogger is introduced to workaround the dependency about logrus
+type ConsoleLogger struct {
+	Enable bool `json:"enable"`
+}
+
+// Loggers struct is defined with FileLogger and ConsoleLogger
+// although they are removed from logging logic. They are
+// kept here just to workaround the dependency migration
+// code/logic has on them.
 type loggers struct {
 	sync.RWMutex
 	Console ConsoleLogger `json:"console"`

cmd/console-logger.go

@@ -1,74 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/Sirupsen/logrus"
-)
-
-// ConsoleLogger - console logger which logs into stderr.
-type ConsoleLogger struct {
-	BaseLogTarget
-}
-
-// Fire - log entry handler.
-func (logger ConsoleLogger) Fire(entry *logrus.Entry) error {
-	if !logger.Enable {
-		return nil
-	}
-
-	msgBytes, err := logger.formatter.Format(entry)
-	if err == nil {
-		fmt.Fprintf(os.Stderr, string(msgBytes))
-	}
-
-	return err
-}
-
-// String - represents ConsoleLogger as string.
-func (logger ConsoleLogger) String() string {
-	enableStr := "disabled"
-	if logger.Enable {
-		enableStr = "enabled"
-	}
-
-	return fmt.Sprintf("console:%s", enableStr)
-}
-
-// NewConsoleLogger - return new console logger object.
-func NewConsoleLogger() (logger ConsoleLogger) {
-	logger.Enable = true
-	logger.formatter = new(logrus.TextFormatter)
-
-	return logger
-}
-
-// InitConsoleLogger - initializes console logger.
-func InitConsoleLogger(logger *ConsoleLogger) {
-	if !logger.Enable {
-		return
-	}
-
-	if logger.formatter == nil {
-		logger.formatter = new(logrus.TextFormatter)
-	}
-
-	return
-}

cmd/erasure-createfile_test.go

@@ -70,7 +70,7 @@ func TestErasureCreateFile(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks)
+		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -125,7 +125,7 @@ func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b
 		b.Fatalf("failed to create test setup: %v", err)
 	}
 	defer setup.Remove()
-	storage, err := NewErasureStorage(setup.disks, data, parity)
+	storage, err := NewErasureStorage(setup.disks, data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}

cmd/erasure-healfile_test.go

@@ -74,7 +74,7 @@ func TestErasureHealFile(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
 		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.disks-test.dataBlocks)
+		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)

cmd/erasure-readfile_test.go

@@ -86,7 +86,7 @@ func TestErasureReadFile(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks)
+		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -174,7 +174,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	}
 	defer setup.Remove()
 
-	storage, err := NewErasureStorage(setup.disks, dataBlocks, parityBlocks)
+	storage, err := NewErasureStorage(setup.disks, dataBlocks, parityBlocks, blockSize)
 	if err != nil {
 		t.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -231,7 +231,7 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b
 		b.Fatalf("failed to create test setup: %v", err)
 	}
 	defer setup.Remove()
-	storage, err := NewErasureStorage(setup.disks, data, parity)
+	storage, err := NewErasureStorage(setup.disks, data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}

cmd/erasure.go

@@ -44,8 +44,9 @@ type ErasureStorage struct {
 
 // NewErasureStorage creates a new ErasureStorage. The storage erasure codes and protects all data written to
 // the disks.
-func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int) (s ErasureStorage, err error) {
-	erasure, err := reedsolomon.New(dataBlocks, parityBlocks)
+func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int, blockSize int64) (s ErasureStorage, err error) {
+	shardsize := (int(blockSize) + dataBlocks - 1) / dataBlocks
+	erasure, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(shardsize))
 	if err != nil {
 		return s, errors.Tracef("failed to create erasure coding: %v", err)
 	}

cmd/erasure_test.go

@@ -52,7 +52,7 @@ func TestErasureDecode(t *testing.T) {
 		copy(buffer, data)
 
 		disks := make([]StorageAPI, test.dataBlocks+test.parityBlocks)
-		storage, err := NewErasureStorage(disks, test.dataBlocks, test.parityBlocks)
+		storage, err := NewErasureStorage(disks, test.dataBlocks, test.parityBlocks, blockSizeV1)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create erasure storage: %v", i, err)
 		}

cmd/event-notifier.go

@@ -365,15 +365,8 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
 	// Construct the notification config path.
 	ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
 
-	// Acquire a write lock on notification config before modifying.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
-	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
-		return nil, err
-	}
-	defer objLock.RUnlock()
-
 	var buffer bytes.Buffer
-	err := objAPI.GetObject(minioMetaBucket, ncPath, 0, -1, &buffer) // Read everything.
+	err := objAPI.GetObject(minioMetaBucket, ncPath, 0, -1, &buffer, "") // Read everything.
 	if err != nil {
 		// 'notification.xml' not found return
 		// 'errNoSuchNotifications'.  This is default when no
@@ -416,15 +409,8 @@ func loadListenerConfig(bucket string, objAPI ObjectLayer) ([]listenerConfig, er
 	// Construct the notification config path.
 	lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
 
-	// Acquire a write lock on notification config before modifying.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
-	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
-		return nil, err
-	}
-	defer objLock.RUnlock()
-
 	var buffer bytes.Buffer
-	err := objAPI.GetObject(minioMetaBucket, lcPath, 0, -1, &buffer)
+	err := objAPI.GetObject(minioMetaBucket, lcPath, 0, -1, &buffer, "")
 	if err != nil {
 		// 'listener.json' not found return
 		// 'errNoSuchNotifications'.  This is default when no
@@ -463,12 +449,6 @@ func persistNotificationConfig(bucket string, ncfg *notificationConfig, obj Obje
 
 	// build path
 	ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
-	// Acquire a write lock on notification config before modifying.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
-	if err = objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
 
 	// write object to path
 	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
@@ -494,12 +474,6 @@ func persistListenerConfig(bucket string, lcfg []listenerConfig, obj ObjectLayer
 
 	// build path
 	lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
-	// Acquire a write lock on notification config before modifying.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
-	if err = objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
 
 	// write object to path
 	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
@@ -526,12 +500,6 @@ func removeNotificationConfig(bucket string, objAPI ObjectLayer) error {
 
 	ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
 
-	// Acquire a write lock on notification config before modifying.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
 	return objAPI.DeleteObject(minioMetaBucket, ncPath)
 }
 
@@ -540,12 +508,6 @@ func removeListenerConfig(bucket string, objAPI ObjectLayer) error {
 	// make the path
 	lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
 
-	// Acquire a write lock on notification config before modifying.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
 	return objAPI.DeleteObject(minioMetaBucket, lcPath)
 }
 

cmd/file-logger.go

@@ -1,86 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/Sirupsen/logrus"
-)
-
-// FileLogger - file logger which logs to a file.
-type FileLogger struct {
-	BaseLogTarget
-	Filename string `json:"filename"`
-	file     *os.File
-}
-
-// Fire - log entry handler.
-func (logger FileLogger) Fire(entry *logrus.Entry) (err error) {
-	if !logger.Enable {
-		return nil
-	}
-
-	msgBytes, err := logger.formatter.Format(entry)
-	if err != nil {
-		return err
-	}
-
-	if _, err = logger.file.Write(msgBytes); err != nil {
-		return err
-	}
-
-	err = logger.file.Sync()
-	return err
-}
-
-// String - represents ConsoleLogger as string.
-func (logger FileLogger) String() string {
-	enableStr := "disabled"
-	if logger.Enable {
-		enableStr = "enabled"
-	}
-
-	return fmt.Sprintf("file:%s:%s", enableStr, logger.Filename)
-}
-
-// NewFileLogger - creates new file logger object.
-func NewFileLogger(filename string) (logger FileLogger) {
-	logger.Enable = true
-	logger.formatter = new(logrus.JSONFormatter)
-	logger.Filename = filename
-
-	return logger
-}
-
-// InitFileLogger - initializes file logger.
-func InitFileLogger(logger *FileLogger) (err error) {
-	if !logger.Enable {
-		return err
-	}
-
-	if logger.formatter == nil {
-		logger.formatter = new(logrus.JSONFormatter)
-	}
-
-	if logger.file == nil {
-		logger.file, err = os.OpenFile(logger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0664)
-	}
-
-	return err
-}

cmd/format-fs.go

@@ -0,0 +1,226 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	errors2 "github.com/minio/minio/pkg/errors"
+	"github.com/minio/minio/pkg/lock"
+)
+
+// FS format version strings.
+const (
+	formatBackendFS   = "fs"
+	formatFSVersionV1 = "1"
+)
+
+// formatFSV1 - structure holds format version '1'.
+type formatFSV1 struct {
+	formatMetaV1
+	FS struct {
+		Version string `json:"version"`
+	} `json:"fs"`
+}
+
+// Used to detect the version of "fs" format.
+type formatFSVersionDetect struct {
+	FS struct {
+		Version string `json:"version"`
+	} `json:"fs"`
+}
+
+// Returns the latest "fs" format.
+func newFormatFSV1() (format *formatFSV1) {
+	f := &formatFSV1{}
+	f.Version = formatMetaVersionV1
+	f.Format = formatBackendFS
+	f.FS.Version = formatFSVersionV1
+	return f
+}
+
+// Save to fs format.json
+func formatFSSave(f *os.File, data interface{}) error {
+	b, err := json.Marshal(data)
+	if err != nil {
+		return errors2.Trace(err)
+	}
+	if err = f.Truncate(0); err != nil {
+		return errors2.Trace(err)
+	}
+	if _, err = f.Seek(0, io.SeekStart); err != nil {
+		return err
+	}
+	_, err = f.Write(b)
+	if err != nil {
+		return errors2.Trace(err)
+	}
+	return nil
+}
+
+// Returns the field formatMetaV1.Format i.e the string "fs" which is never likely to change.
+// We do not use this function in XL to get the format as the file is not fcntl-locked on XL.
+func formatMetaGetFormatBackendFS(r io.ReadSeeker) (string, error) {
+	format := &formatMetaV1{}
+	if err := jsonLoadFromSeeker(r, format); err != nil {
+		return "", err
+	}
+	if format.Version == formatMetaVersionV1 {
+		return format.Format, nil
+	}
+	return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, format.Version)
+}
+
+// Returns formatFS.FS.Version
+func formatFSGetVersion(r io.ReadSeeker) (string, error) {
+	format := &formatFSVersionDetect{}
+	if err := jsonLoadFromSeeker(r, format); err != nil {
+		return "", err
+	}
+	return format.FS.Version, nil
+}
+
+// Migrate the "fs" backend.
+// Migration should happen when formatFSV1.FS.Version changes. This version
+// can change when there is a change to the struct formatFSV1.FS or if there
+// is any change in the backend file system tree structure.
+func formatFSMigrate(wlk *lock.LockedFile) error {
+	// Add any migration code here in case we bump format.FS.Version
+
+	// Make sure that the version is what we expect after the migration.
+	version, err := formatFSGetVersion(wlk)
+	if err != nil {
+		return err
+	}
+	if version != formatFSVersionV1 {
+		return fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV1, version)
+	}
+	return nil
+}
+
+// Creates a new format.json if unformatted.
+func createFormatFS(fsFormatPath string) error {
+	// Attempt a write lock on formatConfigFile `format.json`
+	// file stored in minioMetaBucket(.minio.sys) directory.
+	lk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		return errors2.Trace(err)
+	}
+	// Close the locked file upon return.
+	defer lk.Close()
+
+	fi, err := lk.Stat()
+	if err != nil {
+		return errors2.Trace(err)
+	}
+	if fi.Size() != 0 {
+		// format.json already got created because of another minio process's createFormatFS()
+		return nil
+	}
+
+	return formatFSSave(lk.File, newFormatFSV1())
+}
+
+// This function returns a read-locked format.json reference to the caller.
+// The file descriptor should be kept open throughout the life
+// of the process so that another minio process does not try to
+// migrate the backend when we are actively working on the backend.
+func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) {
+	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
+	// Any read on format.json should be done with read-lock.
+	// Any write on format.json should be done with write-lock.
+	for {
+		isEmpty := false
+		rlk, err := lock.RLockedOpenFile(fsFormatPath)
+		if err == nil {
+			// format.json can be empty in a rare condition when another
+			// minio process just created the file but could not hold lock
+			// and write to it.
+			var fi os.FileInfo
+			fi, err = rlk.Stat()
+			if err != nil {
+				return nil, errors2.Trace(err)
+			}
+			isEmpty = fi.Size() == 0
+		}
+		if os.IsNotExist(err) || isEmpty {
+			if err == nil {
+				rlk.Close()
+			}
+			// Fresh disk - create format.json
+			err = createFormatFS(fsFormatPath)
+			if err == lock.ErrAlreadyLocked {
+				// Lock already present, sleep and attempt again.
+				// Can happen in a rare situation when a parallel minio process
+				// holds the lock and creates format.json
+				time.Sleep(100 * time.Millisecond)
+				continue
+			}
+			if err != nil {
+				return nil, errors2.Trace(err)
+			}
+			// After successfully creating format.json try to hold a read-lock on
+			// the file.
+			continue
+		}
+		if err != nil {
+			return nil, errors2.Trace(err)
+		}
+
+		formatBackend, err := formatMetaGetFormatBackendFS(rlk)
+		if err != nil {
+			return nil, errors2.Trace(err)
+		}
+		if formatBackend != formatBackendFS {
+			return nil, fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend)
+		}
+		version, err := formatFSGetVersion(rlk)
+		if err != nil {
+			return nil, err
+		}
+		if version != formatFSVersionV1 {
+			// Format needs migration
+			rlk.Close()
+			// Hold write lock during migration so that we do not disturb any
+			// minio processes running in parallel.
+			wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+			if err == lock.ErrAlreadyLocked {
+				// Lock already present, sleep and attempt again.
+				wlk.Close()
+				time.Sleep(100 * time.Millisecond)
+				continue
+			}
+			if err != nil {
+				return nil, err
+			}
+			err = formatFSMigrate(wlk)
+			wlk.Close()
+			if err != nil {
+				// Migration failed, bail out so that the user can observe what happened.
+				return nil, err
+			}
+			// Successfully migrated, now try to hold a read-lock on format.json
+			continue
+		}
+
+		return rlk, nil
+	}
+}

cmd/format-fs_test.go

@@ -0,0 +1,102 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestFSFormatFS - tests initFormatFS, formatMetaGetFormatBackendFS, formatFSGetVersion.
+func TestFSFormatFS(t *testing.T) {
+	// Prepare for testing
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(disk)
+
+	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
+
+	// Assign a new UUID.
+	uuid := mustGetUUID()
+
+	// Initialize meta volume, if volume already exists ignores it.
+	if err := initMetaVolumeFS(disk, uuid); err != nil {
+		t.Fatal(err)
+	}
+
+	rlk, err := initFormatFS(disk)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rlk.Close()
+
+	// Do the basic sanity checks to check if initFormatFS() did its job.
+	f, err := os.OpenFile(fsFormatPath, os.O_RDWR, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	format, err := formatMetaGetFormatBackendFS(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if format != formatBackendFS {
+		t.Fatalf(`expected: %s, got: %s`, formatBackendFS, format)
+	}
+	version, err := formatFSGetVersion(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if version != formatFSVersionV1 {
+		t.Fatalf(`expected: %s, got: %s`, formatFSVersionV1, version)
+	}
+
+	// Corrupt the format.json file and test the functions.
+	// formatMetaGetFormatBackendFS, formatFSGetVersion, initFormatFS should return errors.
+	if err = f.Truncate(0); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = f.WriteString("b"); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = formatMetaGetFormatBackendFS(f); err == nil {
+		t.Fatal("expected to fail")
+	}
+	if _, err = formatFSGetVersion(rlk); err == nil {
+		t.Fatal("expected to fail")
+	}
+	if _, err = initFormatFS(disk); err == nil {
+		t.Fatal("expected to fail")
+	}
+
+	// With unknown formatMetaV1.Version formatMetaGetFormatBackendFS, initFormatFS should return error.
+	if err = f.Truncate(0); err != nil {
+		t.Fatal(err)
+	}
+	// Here we set formatMetaV1.Version to "2"
+	if _, err = f.WriteString(`{"version":"2","format":"fs","fs":{"version":"1"}}`); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = formatMetaGetFormatBackendFS(f); err == nil {
+		t.Fatal("expected to fail")
+	}
+	if _, err = initFormatFS(disk); err == nil {
+		t.Fatal("expected to fail")
+	}
+}

cmd/format-meta.go

@@ -0,0 +1,52 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// Format related consts
+const (
+	// Format config file carries backend format specific details.
+	formatConfigFile = "format.json"
+
+	// Format config tmp file carries backend format.
+	formatConfigFileTmp = "format.json.tmp"
+)
+
+const (
+	// Version of the formatMetaV1
+	formatMetaVersionV1 = "1"
+)
+
+// format.json currently has the format:
+// {
+//   "version": "1",
+//   "format": "XXXXX",
+//   "XXXXX": {
+//
+//   }
+// }
+// Here "XXXXX" depends on the backend, currently we have "fs" and "xl" implementations.
+// formatMetaV1 should be inherited by backend format structs. Please look at format-fs.go
+// and format-xl.go for details.
+
+// Ideally we will never have a situation where we will have to change the
+// fields of this struct and deal with related migration.
+type formatMetaV1 struct {
+	// Version of the format config.
+	Version string `json:"version"`
+	// Format indicates the backend format type, supports two values 'xl' and 'fs'.
+	Format string `json:"format"`
+}

cmd/format-xl.go

@@ -20,168 +20,30 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"reflect"
 	"sync"
-
-	errors2 "github.com/minio/minio/pkg/errors"
-	"github.com/minio/minio/pkg/lock"
 )
 
-// fsFormat - structure holding 'fs' format.
-type fsFormat struct {
-	Version string `json:"version"`
-}
-
-// FS format version strings.
 const (
-	// Represents the current backend disk structure
-	// version under `.minio.sys` and actual data namespace.
-
-	// formatConfigV1.fsFormat.Version
-	fsFormatBackendV1 = "1"
-)
-
-// xlFormat - structure holding 'xl' format.
-type xlFormat struct {
-	Version string `json:"version"` // Version of 'xl' format.
-	Disk    string `json:"disk"`    // Disk field carries assigned disk uuid.
-	// JBOD field carries the input disk order generated the first
-	// time when fresh disks were supplied.
-	JBOD []string `json:"jbod"`
-}
-
-// XL format version strings.
-const (
-	// Represents the current backend disk structure
-	// version under `.minio.sys` and actual data namespace.
-
-	// formatConfigV1.xlFormat.Version
-	xlFormatBackendV1 = "1"
-)
-
-// formatConfigV1 - structure holds format config version '1'.
-type formatConfigV1 struct {
-	Version string `json:"version"` // Version of the format config.
-	// Format indicates the backend format type, supports two values 'xl' and 'fs'.
-	Format string    `json:"format"`
-	FS     *fsFormat `json:"fs,omitempty"` // FS field holds fs format.
-	XL     *xlFormat `json:"xl,omitempty"` // XL field holds xl format.
-}
-
-// Format json file.
-const (
-	// Format config file carries backend format specific details.
-	formatConfigFile = "format.json"
-
-	// Format config tmp file carries backend format.
-	formatConfigFileTmp = "format.json.tmp"
-)
-
-// `format.json` version value.
-const (
-	// formatConfigV1.Version represents the version string
-	// of the current structure and its fields in `format.json`.
-	formatFileV1 = "1"
-
-	// Future `format.json` structure changes should have
-	// its own version and should be subsequently listed here.
-)
-
-// Constitutes `format.json` backend name.
-const (
-	// Represents FS backend.
-	formatBackendFS = "fs"
-
 	// Represents XL backend.
 	formatBackendXL = "xl"
+
+	// formatXLV1.XL.Version
+	formatXLVersionV1 = "1"
 )
 
-// CheckFS if the format is FS and is valid with right values
-// returns appropriate errors otherwise.
-func (f *formatConfigV1) CheckFS() error {
-	// Validate if format config version is v1.
-	if f.Version != formatFileV1 {
-		return fmt.Errorf("Unknown format file version '%s'", f.Version)
-	}
-
-	// Validate if we have the expected format.
-	if f.Format != formatBackendFS {
-		return fmt.Errorf("FS backend format required. Found '%s'", f.Format)
-	}
-
-	// Check if format is currently supported.
-	if f.FS.Version != fsFormatBackendV1 {
-		return fmt.Errorf("Unknown backend FS format version '%s'", f.FS.Version)
-	}
-
-	// Success.
-	return nil
-}
-
-// LoadFormat - loads format config v1, returns `errUnformattedDisk`
-// if reading format.json fails with io.EOF.
-func (f *formatConfigV1) LoadFormat(lk *lock.LockedFile) error {
-	_, err := f.ReadFrom(lk)
-	if errors2.Cause(err) == io.EOF {
-		// No data on disk `format.json` still empty
-		// treat it as unformatted disk.
-		return errors2.Trace(errUnformattedDisk)
-	}
-	return err
-}
-
-func (f *formatConfigV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
-	// Serialize to prepare to write to disk.
-	var fbytes []byte
-	fbytes, err = json.Marshal(f)
-	if err != nil {
-		return 0, errors2.Trace(err)
-	}
-	if err = lk.Truncate(0); err != nil {
-		return 0, errors2.Trace(err)
-	}
-	_, err = lk.Write(fbytes)
-	if err != nil {
-		return 0, errors2.Trace(err)
-	}
-	return int64(len(fbytes)), nil
-}
-
-func (f *formatConfigV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
-	var fbytes []byte
-	fi, err := lk.Stat()
-	if err != nil {
-		return 0, errors2.Trace(err)
-	}
-	fbytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
-	if err != nil {
-		return 0, errors2.Trace(err)
-	}
-	if len(fbytes) == 0 {
-		return 0, errors2.Trace(io.EOF)
-	}
-	// Decode `format.json`.
-	if err = json.Unmarshal(fbytes, f); err != nil {
-		return 0, errors2.Trace(err)
-	}
-	return int64(len(fbytes)), nil
-}
-
-func newFSFormat() (format *formatConfigV1) {
-	return newFSFormatV1()
-}
-
-// newFSFormatV1 - initializes new formatConfigV1 with FS format info.
-func newFSFormatV1() (format *formatConfigV1) {
-	return &formatConfigV1{
-		Version: formatFileV1,
-		Format:  formatBackendFS,
-		FS: &fsFormat{
-			Version: fsFormatBackendV1,
-		},
-	}
+// Represents the current backend disk structure
+// version under `.minio.sys` and actual data namespace.
+// formatXLV1 - structure holds format config version '1'.
+type formatXLV1 struct {
+	formatMetaV1
+	XL struct {
+		Version string `json:"version"` // Version of 'xl' format.
+		Disk    string `json:"disk"`    // Disk field carries assigned disk uuid.
+		// JBOD field carries the input disk order generated the first
+		// time when fresh disks were supplied.
+		JBOD []string `json:"jbod"`
+	} `json:"xl"` // XL field holds xl format.
 }
 
 /*
@@ -271,7 +133,7 @@ func formatErrsSummary(errs []error) (formatCount, unformattedDiskCount,
 }
 
 // loadAllFormats - load all format config from all input disks in parallel.
-func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
+func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatXLV1, []error) {
 	// Initialize sync waitgroup.
 	var wg = &sync.WaitGroup{}
 
@@ -279,7 +141,7 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
 	var sErrs = make([]error, len(bootstrapDisks))
 
 	// Initialize format configs.
-	var formatConfigs = make([]*formatConfigV1, len(bootstrapDisks))
+	var formats = make([]*formatXLV1, len(bootstrapDisks))
 
 	// Make a volume entry on all underlying storage disks.
 	for index, disk := range bootstrapDisks {
@@ -291,12 +153,12 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
 		// Make a volume inside a go-routine.
 		go func(index int, disk StorageAPI) {
 			defer wg.Done()
-			formatConfig, lErr := loadFormat(disk)
+			format, lErr := loadFormat(disk)
 			if lErr != nil {
 				sErrs[index] = lErr
 				return
 			}
-			formatConfigs[index] = formatConfig
+			formats[index] = format
 		}(index, disk)
 	}
 
@@ -306,11 +168,11 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
 	for _, err := range sErrs {
 		if err != nil {
 			// Return all formats and errors.
-			return formatConfigs, sErrs
+			return formats, sErrs
 		}
 	}
 	// Return all formats and nil
-	return formatConfigs, sErrs
+	return formats, sErrs
 }
 
 // genericFormatCheckXL - validates and returns error.
@@ -318,7 +180,7 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
 // if (any disk is corrupt) return error // phase2
 // if (jbod inconsistent) return error // phase2
 // if (disks not recognized) // Always error.
-func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err error) {
+func genericFormatCheckXL(formats []*formatXLV1, sErrs []error) (err error) {
 	// Calculate the errors.
 	var (
 		errCorruptFormatCount = 0
@@ -342,20 +204,20 @@ func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err e
 	}
 
 	// Calculate read quorum.
-	readQuorum := len(formatConfigs) / 2
+	readQuorum := len(formats) / 2
 
 	// Validate the err count under read quorum.
-	if errCount > len(formatConfigs)-readQuorum {
+	if errCount > len(formats)-readQuorum {
 		return errXLReadQuorum
 	}
 
 	// Check if number of corrupted format under read quorum
-	if errCorruptFormatCount > len(formatConfigs)-readQuorum {
+	if errCorruptFormatCount > len(formats)-readQuorum {
 		return errCorruptedFormat
 	}
 
 	// Validates if format and JBOD are consistent across all disks.
-	if err = checkFormatXL(formatConfigs); err != nil {
+	if err = checkFormatXL(formats); err != nil {
 		return err
 	}
 
@@ -366,15 +228,15 @@ func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err e
 // isSavedUUIDInOrder - validates if disk uuid is present and valid in all
 // available format config JBOD. This function also validates if the disk UUID
 // is always available on all JBOD under the same order.
-func isSavedUUIDInOrder(uuid string, formatConfigs []*formatConfigV1) bool {
+func isSavedUUIDInOrder(uuid string, formats []*formatXLV1) bool {
 	var orderIndexes []int
 	// Validate each for format.json for relevant uuid.
-	for _, formatConfig := range formatConfigs {
-		if formatConfig == nil {
+	for _, format := range formats {
+		if format == nil {
 			continue
 		}
 		// Validate if UUID is present in JBOD.
-		uuidIndex := findDiskIndex(uuid, formatConfig.XL.JBOD)
+		uuidIndex := findDiskIndex(uuid, format.XL.JBOD)
 		if uuidIndex == -1 {
 			// UUID not found.
 			errorIf(errDiskNotFound, "Disk %s not found in JBOD list", uuid)
@@ -398,15 +260,15 @@ func isSavedUUIDInOrder(uuid string, formatConfigs []*formatConfigV1) bool {
 }
 
 // checkDisksConsistency - checks if all disks are consistent with all JBOD entries on all disks.
-func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
-	var disks = make([]string, len(formatConfigs))
+func checkDisksConsistency(formats []*formatXLV1) error {
+	var disks = make([]string, len(formats))
 	// Collect currently available disk uuids.
-	for index, formatConfig := range formatConfigs {
-		if formatConfig == nil {
+	for index, format := range formats {
+		if format == nil {
 			disks[index] = ""
 			continue
 		}
-		disks[index] = formatConfig.XL.Disk
+		disks[index] = format.XL.Disk
 	}
 	// Validate collected uuids and verify JBOD.
 	for _, uuid := range disks {
@@ -414,7 +276,7 @@ func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
 			continue
 		}
 		// Is uuid present on all JBOD ?.
-		if !isSavedUUIDInOrder(uuid, formatConfigs) {
+		if !isSavedUUIDInOrder(uuid, formats) {
 			return fmt.Errorf("%s disk not found in JBOD", uuid)
 		}
 	}
@@ -422,17 +284,17 @@ func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
 }
 
 // checkJBODConsistency - validate xl jbod order if they are consistent.
-func checkJBODConsistency(formatConfigs []*formatConfigV1) error {
+func checkJBODConsistency(formats []*formatXLV1) error {
 	var sentinelJBOD []string
 	// Extract first valid JBOD.
-	for _, format := range formatConfigs {
+	for _, format := range formats {
 		if format == nil {
 			continue
 		}
 		sentinelJBOD = format.XL.JBOD
 		break
 	}
-	for _, format := range formatConfigs {
+	for _, format := range formats {
 		if format == nil {
 			continue
 		}
@@ -458,14 +320,14 @@ func findDiskIndex(disk string, jbod []string) int {
 // format-config. If assignUUIDs is true, it assigns UUIDs to disks
 // with missing format configurations in the reference configuration.
 func reorderDisks(bootstrapDisks []StorageAPI,
-	formatConfigs []*formatConfigV1, assignUUIDs bool) (*formatConfigV1,
+	formats []*formatXLV1, assignUUIDs bool) (*formatXLV1,
 	[]StorageAPI, error) {
 
 	// Pick first non-nil format-cfg as reference
-	var refCfg *formatConfigV1
-	for _, formatConfig := range formatConfigs {
-		if formatConfig != nil {
-			refCfg = formatConfig
+	var refCfg *formatXLV1
+	for _, format := range formats {
+		if format != nil {
+			refCfg = format
 			break
 		}
 	}
@@ -476,7 +338,7 @@ func reorderDisks(bootstrapDisks []StorageAPI,
 
 	// construct reordered disk slice
 	var newDisks = make([]StorageAPI, len(bootstrapDisks))
-	for fIndex, format := range formatConfigs {
+	for fIndex, format := range formats {
 		if format == nil {
 			continue
 		}
@@ -501,7 +363,7 @@ func reorderDisks(bootstrapDisks []StorageAPI,
 }
 
 // loadFormat - loads format.json from disk.
-func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
+func loadFormat(disk StorageAPI) (format *formatXLV1, err error) {
 	buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
 	if err != nil {
 		// 'file not found' and 'volume not found' as
@@ -523,7 +385,7 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
 	}
 
 	// Try to decode format json into formatConfigV1 struct.
-	format = &formatConfigV1{}
+	format = &formatXLV1{}
 	if err = json.Unmarshal(buf, format); err != nil {
 		return nil, err
 	}
@@ -535,23 +397,20 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
 // collectNSaveNewFormatConfigs - creates new format configs based on
 // the reference config and saves it on all disks, this is to be
 // called from healFormatXL* functions.
-func collectNSaveNewFormatConfigs(referenceConfig *formatConfigV1,
+func collectNSaveNewFormatConfigs(referenceConfig *formatXLV1,
 	orderedDisks []StorageAPI) error {
 
 	// Collect new format configs that need to be written.
-	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
+	var newFormatConfigs = make([]*formatXLV1, len(orderedDisks))
 	for index := range orderedDisks {
 		// New configs are generated since we are going
 		// to re-populate across all disks.
-		config := &formatConfigV1{
-			Version: referenceConfig.Version,
-			Format:  referenceConfig.Format,
-			XL: &xlFormat{
-				Version: referenceConfig.XL.Version,
-				Disk:    referenceConfig.XL.JBOD[index],
-				JBOD:    referenceConfig.XL.JBOD,
-			},
-		}
+		config := &formatXLV1{}
+		config.Version = referenceConfig.Version
+		config.Format = referenceConfig.Format
+		config.XL.Version = referenceConfig.XL.Version
+		config.XL.Disk = referenceConfig.XL.JBOD[index]
+		config.XL.JBOD = referenceConfig.XL.JBOD
 		newFormatConfigs[index] = config
 	}
 
@@ -568,11 +427,11 @@ func collectNSaveNewFormatConfigs(referenceConfig *formatConfigV1,
 // unexpected errors as regular errors can be ignored since there
 // might be enough quorum to be operational.  Heals only fresh disks.
 func healFormatXLFreshDisks(storageDisks []StorageAPI,
-	formatConfigs []*formatConfigV1) error {
+	formats []*formatXLV1) error {
 
 	// Reorder the disks based on the JBOD order.
 	referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
-		formatConfigs, true)
+		formats, true)
 	if err != nil {
 		return err
 	}
@@ -581,7 +440,7 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI,
 	// We need to make sure we have kept the previous order
 	// and allowed fresh disks to be arranged anywhere.
 	// Following block facilitates to put fresh disks.
-	for index, format := range formatConfigs {
+	for index, format := range formats {
 		// Format is missing so we go through ordered disks.
 		if format == nil {
 			// At this point when disk is missing the fresh disk
@@ -624,9 +483,9 @@ func collectUnAssignedDisks(storageDisks, orderedDisks []StorageAPI) (
 // Inspect the content of all disks to guess the right order according
 // to the format files. The right order is represented in orderedDisks
 func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI,
-	formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
+	formats []*formatXLV1) ([]StorageAPI, error) {
 
-	for index, format := range formatConfigs {
+	for index, format := range formats {
 		if format != nil {
 			continue
 		}
@@ -681,11 +540,11 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI,
 
 // Heals corrupted format json in all disks
 func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
-	formatConfigs []*formatConfigV1) error {
+	formats []*formatXLV1) error {
 
 	// Reorder the disks based on the JBOD order.
 	referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
-		formatConfigs, true)
+		formats, true)
 	if err != nil {
 		return err
 	}
@@ -693,7 +552,7 @@ func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
 	// For disks with corrupted formats, inspect the disks
 	// contents to guess the disks order
 	orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks,
-		formatConfigs)
+		formats)
 	if err != nil {
 		return err
 	}
@@ -721,7 +580,7 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
 	var unformattedDisksFoundCnt = 0
 	var diskNotFoundCount = 0
 	var corruptedDisksFoundCnt = 0
-	formatConfigs := make([]*formatConfigV1, len(bootstrapDisks))
+	formats := make([]*formatXLV1, len(bootstrapDisks))
 
 	// Try to load `format.json` bootstrap disks.
 	for index, disk := range bootstrapDisks {
@@ -729,7 +588,7 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
 			diskNotFoundCount++
 			continue
 		}
-		var formatXL *formatConfigV1
+		var formatXL *formatXLV1
 		formatXL, err = loadFormat(disk)
 		if err != nil {
 			if err == errUnformattedDisk {
@@ -745,7 +604,7 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
 			return nil, err
 		}
 		// Save valid formats.
-		formatConfigs[index] = formatXL
+		formats[index] = formatXL
 	}
 
 	// If all disks indicate that 'format.json' is not available return 'errUnformattedDisk'.
@@ -760,59 +619,59 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
 	}
 
 	// Validate the format configs read are correct.
-	if err = checkFormatXL(formatConfigs); err != nil {
+	if err = checkFormatXL(formats); err != nil {
 		return nil, err
 	}
 	// Erasure code requires disks to be presented in the same
 	// order each time.
-	_, orderedDisks, err := reorderDisks(bootstrapDisks, formatConfigs,
+	_, orderedDisks, err := reorderDisks(bootstrapDisks, formats,
 		false)
 	return orderedDisks, err
 }
 
-func checkFormatXLValue(formatXL *formatConfigV1) error {
+func checkFormatXLValue(formatXL *formatXLV1) error {
 	// Validate format version and format type.
-	if formatXL.Version != formatFileV1 {
+	if formatXL.Version != formatMetaVersionV1 {
 		return fmt.Errorf("Unsupported version of backend format [%s] found", formatXL.Version)
 	}
 	if formatXL.Format != formatBackendXL {
 		return fmt.Errorf("Unsupported backend format [%s] found", formatXL.Format)
 	}
-	if formatXL.XL.Version != "1" {
+	if formatXL.XL.Version != formatXLVersionV1 {
 		return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version)
 	}
 	return nil
 }
 
-func checkFormatXLValues(formatConfigs []*formatConfigV1) (int, error) {
-	for i, formatXL := range formatConfigs {
+func checkFormatXLValues(formats []*formatXLV1) (int, error) {
+	for i, formatXL := range formats {
 		if formatXL == nil {
 			continue
 		}
 		if err := checkFormatXLValue(formatXL); err != nil {
 			return i, err
 		}
-		if len(formatConfigs) != len(formatXL.XL.JBOD) {
+		if len(formats) != len(formatXL.XL.JBOD) {
 			return i, fmt.Errorf("Number of disks %d did not match the backend format %d",
-				len(formatConfigs), len(formatXL.XL.JBOD))
+				len(formats), len(formatXL.XL.JBOD))
 		}
 	}
 	return -1, nil
 }
 
 // checkFormatXL - verifies if format.json format is intact.
-func checkFormatXL(formatConfigs []*formatConfigV1) error {
-	if _, err := checkFormatXLValues(formatConfigs); err != nil {
+func checkFormatXL(formats []*formatXLV1) error {
+	if _, err := checkFormatXLValues(formats); err != nil {
 		return err
 	}
-	if err := checkJBODConsistency(formatConfigs); err != nil {
+	if err := checkJBODConsistency(formats); err != nil {
 		return err
 	}
-	return checkDisksConsistency(formatConfigs)
+	return checkDisksConsistency(formats)
 }
 
 // saveFormatXL - populates `format.json` on disks in its order.
-func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error {
+func saveFormatXL(storageDisks []StorageAPI, formats []*formatXLV1) error {
 	var errs = make([]error, len(storageDisks))
 	var wg = &sync.WaitGroup{}
 	// Write `format.json` to all disks.
@@ -821,7 +680,7 @@ func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error {
 			continue
 		}
 		wg.Add(1)
-		go func(index int, disk StorageAPI, format *formatConfigV1) {
+		go func(index int, disk StorageAPI, format *formatXLV1) {
 			defer wg.Done()
 
 			// Marshal and write to disk.
@@ -862,45 +721,37 @@ func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error {
 	return nil
 }
 
-// initFormatXL - save XL format configuration on all disks.
-func initFormatXL(storageDisks []StorageAPI) (err error) {
-	// Initialize jbods.
-	var jbod = make([]string, len(storageDisks))
+// Return a slice of format, to be used to format uninitialized disks.
+func newFormatXLV1(diskCount int) []*formatXLV1 {
+	var jbod = make([]string, diskCount)
 
-	// Initialize formats.
-	var formats = make([]*formatConfigV1, len(storageDisks))
+	var formats = make([]*formatXLV1, diskCount)
 
-	// Initialize `format.json`.
-	for index, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-		// Allocate format config.
-		formats[index] = &formatConfigV1{
-			Version: formatFileV1,
-			Format:  formatBackendXL,
-			XL: &xlFormat{
-				Version: xlFormatBackendV1,
-				Disk:    mustGetUUID(),
-			},
-		}
-		jbod[index] = formats[index].XL.Disk
+	for i := 0; i < diskCount; i++ {
+		format := &formatXLV1{}
+		format.Version = formatMetaVersionV1
+		format.Format = formatBackendXL
+		format.XL.Version = formatXLVersionV1
+		format.XL.Disk = mustGetUUID()
+		formats[i] = format
+		jbod[i] = formats[i].XL.Disk
 	}
 
 	// Update the jbod entries.
-	for index, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-		// Save jbod.
-		formats[index].XL.JBOD = jbod
+	for i := 0; i < diskCount; i++ {
+		formats[i].XL.JBOD = jbod
 	}
 
+	return formats
+}
+
+// initFormatXL - save XL format configuration on all disks.
+func initFormatXL(storageDisks []StorageAPI) (err error) {
 	// Initialize meta volume, if volume already exists ignores it.
 	if err := initMetaVolume(storageDisks); err != nil {
 		return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
 	}
 
 	// Save formats `format.json` across all disks.
-	return saveFormatXL(storageDisks, formats)
+	return saveFormatXL(storageDisks, newFormatXLV1(len(storageDisks)))
 }

cmd/format-xl_test.go

@@ -18,54 +18,46 @@ package cmd
 
 import (
 	"bytes"
-	"errors"
 	"os"
-	"path/filepath"
 	"testing"
 
-	errors2 "github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/lock"
 )
 
 // generates a valid format.json for XL backend.
-func genFormatXLValid() []*formatConfigV1 {
+func genFormatXLValid() []*formatXLV1 {
 	jbod := make([]string, 8)
-	formatConfigs := make([]*formatConfigV1, 8)
+	formatConfigs := make([]*formatXLV1, 8)
 	for index := range jbod {
 		jbod[index] = mustGetUUID()
 	}
 	for index := range jbod {
-		formatConfigs[index] = &formatConfigV1{
-			Version: formatFileV1,
-			Format:  formatBackendXL,
-			XL: &xlFormat{
-				Version: xlFormatBackendV1,
-				Disk:    jbod[index],
-				JBOD:    jbod,
-			},
-		}
+		format := &formatXLV1{}
+		format.Version = formatMetaVersionV1
+		format.Format = formatBackendXL
+		format.XL.Version = formatXLVersionV1
+		format.XL.Disk = jbod[index]
+		format.XL.JBOD = jbod
+		formatConfigs[index] = format
 	}
 	return formatConfigs
 }
 
 // generates a invalid format.json version for XL backend.
-func genFormatXLInvalidVersion() []*formatConfigV1 {
+func genFormatXLInvalidVersion() []*formatXLV1 {
 	jbod := make([]string, 8)
-	formatConfigs := make([]*formatConfigV1, 8)
+	formatConfigs := make([]*formatXLV1, 8)
 	for index := range jbod {
 		jbod[index] = mustGetUUID()
 	}
 	for index := range jbod {
-		formatConfigs[index] = &formatConfigV1{
-			Version: formatFileV1,
-			Format:  formatBackendXL,
-			XL: &xlFormat{
-				Version: xlFormatBackendV1,
-				Disk:    jbod[index],
-				JBOD:    jbod,
-			},
-		}
+		format := &formatXLV1{}
+		format.Version = formatMetaVersionV1
+		format.Format = formatBackendXL
+		format.XL.Version = formatXLVersionV1
+		format.XL.Disk = jbod[index]
+		format.XL.JBOD = jbod
+		formatConfigs[index] = format
 	}
 	// Corrupt version numbers.
 	formatConfigs[0].Version = "2"
@@ -74,46 +66,42 @@ func genFormatXLInvalidVersion() []*formatConfigV1 {
 }
 
 // generates a invalid format.json version for XL backend.
-func genFormatXLInvalidFormat() []*formatConfigV1 {
+func genFormatXLInvalidFormat() []*formatXLV1 {
 	jbod := make([]string, 8)
-	formatConfigs := make([]*formatConfigV1, 8)
+	formatConfigs := make([]*formatXLV1, 8)
 	for index := range jbod {
 		jbod[index] = mustGetUUID()
 	}
 	for index := range jbod {
-		formatConfigs[index] = &formatConfigV1{
-			Version: formatFileV1,
-			Format:  formatBackendXL,
-			XL: &xlFormat{
-				Version: xlFormatBackendV1,
-				Disk:    jbod[index],
-				JBOD:    jbod,
-			},
-		}
+		format := &formatXLV1{}
+		format.Version = formatMetaVersionV1
+		format.Format = formatBackendXL
+		format.XL.Version = formatXLVersionV1
+		format.XL.Disk = jbod[index]
+		format.XL.JBOD = jbod
+		formatConfigs[index] = format
 	}
-	// Corrupt version numbers.
+	// Corrupt format.
 	formatConfigs[0].Format = "lx"
 	formatConfigs[3].Format = "lx"
 	return formatConfigs
 }
 
 // generates a invalid format.json version for XL backend.
-func genFormatXLInvalidXLVersion() []*formatConfigV1 {
+func genFormatXLInvalidXLVersion() []*formatXLV1 {
 	jbod := make([]string, 8)
-	formatConfigs := make([]*formatConfigV1, 8)
+	formatConfigs := make([]*formatXLV1, 8)
 	for index := range jbod {
 		jbod[index] = mustGetUUID()
 	}
 	for index := range jbod {
-		formatConfigs[index] = &formatConfigV1{
-			Version: formatFileV1,
-			Format:  formatBackendXL,
-			XL: &xlFormat{
-				Version: xlFormatBackendV1,
-				Disk:    jbod[index],
-				JBOD:    jbod,
-			},
-		}
+		format := &formatXLV1{}
+		format.Version = formatMetaVersionV1
+		format.Format = formatBackendXL
+		format.XL.Version = formatXLVersionV1
+		format.XL.Disk = jbod[index]
+		format.XL.JBOD = jbod
+		formatConfigs[index] = format
 	}
 	// Corrupt version numbers.
 	formatConfigs[0].XL.Version = "10"
@@ -121,51 +109,47 @@ func genFormatXLInvalidXLVersion() []*formatConfigV1 {
 	return formatConfigs
 }
 
-func genFormatFS() *formatConfigV1 {
-	return &formatConfigV1{
-		Version: formatFileV1,
-		Format:  formatBackendFS,
-	}
+func genFormatFS() *formatXLV1 {
+	format := &formatXLV1{}
+	format.Version = formatMetaVersionV1
+	format.Format = formatBackendFS
+	return format
 }
 
 // generates a invalid format.json version for XL backend.
-func genFormatXLInvalidJBODCount() []*formatConfigV1 {
+func genFormatXLInvalidJBODCount() []*formatXLV1 {
 	jbod := make([]string, 7)
-	formatConfigs := make([]*formatConfigV1, 8)
+	formatConfigs := make([]*formatXLV1, 8)
 	for index := range jbod {
 		jbod[index] = mustGetUUID()
 	}
 	for index := range jbod {
-		formatConfigs[index] = &formatConfigV1{
-			Version: formatFileV1,
-			Format:  formatBackendXL,
-			XL: &xlFormat{
-				Version: xlFormatBackendV1,
-				Disk:    jbod[index],
-				JBOD:    jbod,
-			},
-		}
+		format := &formatXLV1{}
+		format.Version = formatMetaVersionV1
+		format.Format = formatBackendXL
+		format.XL.Version = formatXLVersionV1
+		format.XL.Disk = jbod[index]
+		format.XL.JBOD = jbod
+		formatConfigs[index] = format
 	}
 	return formatConfigs
 }
 
 // generates a invalid format.json JBOD for XL backend.
-func genFormatXLInvalidJBOD() []*formatConfigV1 {
+func genFormatXLInvalidJBOD() []*formatXLV1 {
 	jbod := make([]string, 8)
-	formatConfigs := make([]*formatConfigV1, 8)
+	formatConfigs := make([]*formatXLV1, 8)
 	for index := range jbod {
 		jbod[index] = mustGetUUID()
 	}
 	for index := range jbod {
-		formatConfigs[index] = &formatConfigV1{
-			Version: formatFileV1,
-			Format:  formatBackendXL,
-			XL: &xlFormat{
-				Version: xlFormatBackendV1,
-				Disk:    jbod[index],
-				JBOD:    jbod,
-			},
-		}
+		format := &formatXLV1{}
+		format.Version = formatMetaVersionV1
+		format.Format = formatBackendXL
+		format.XL.Version = formatXLVersionV1
+		format.XL.Disk = jbod[index]
+		format.XL.JBOD = jbod
+		formatConfigs[index] = format
 	}
 	for index := range jbod {
 		jbod[index] = mustGetUUID()
@@ -177,22 +161,20 @@ func genFormatXLInvalidJBOD() []*formatConfigV1 {
 }
 
 // generates a invalid format.json Disk UUID for XL backend.
-func genFormatXLInvalidDisks() []*formatConfigV1 {
+func genFormatXLInvalidDisks() []*formatXLV1 {
 	jbod := make([]string, 8)
-	formatConfigs := make([]*formatConfigV1, 8)
+	formatConfigs := make([]*formatXLV1, 8)
 	for index := range jbod {
 		jbod[index] = mustGetUUID()
 	}
 	for index := range jbod {
-		formatConfigs[index] = &formatConfigV1{
-			Version: formatFileV1,
-			Format:  formatBackendXL,
-			XL: &xlFormat{
-				Version: xlFormatBackendV1,
-				Disk:    jbod[index],
-				JBOD:    jbod,
-			},
-		}
+		format := &formatXLV1{}
+		format.Version = formatMetaVersionV1
+		format.Format = formatBackendXL
+		format.XL.Version = formatXLVersionV1
+		format.XL.Disk = jbod[index]
+		format.XL.JBOD = jbod
+		formatConfigs[index] = format
 	}
 	// Make disk 5 and disk 8 have inconsistent disk uuid's.
 	formatConfigs[4].XL.Disk = mustGetUUID()
@@ -201,22 +183,20 @@ func genFormatXLInvalidDisks() []*formatConfigV1 {
 }
 
 // generates a invalid format.json Disk UUID in wrong order for XL backend.
-func genFormatXLInvalidDisksOrder() []*formatConfigV1 {
+func genFormatXLInvalidDisksOrder() []*formatXLV1 {
 	jbod := make([]string, 8)
-	formatConfigs := make([]*formatConfigV1, 8)
+	formatConfigs := make([]*formatXLV1, 8)
 	for index := range jbod {
 		jbod[index] = mustGetUUID()
 	}
 	for index := range jbod {
-		formatConfigs[index] = &formatConfigV1{
-			Version: formatFileV1,
-			Format:  formatBackendXL,
-			XL: &xlFormat{
-				Version: xlFormatBackendV1,
-				Disk:    jbod[index],
-				JBOD:    jbod,
-			},
-		}
+		format := &formatXLV1{}
+		format.Version = formatMetaVersionV1
+		format.Format = formatBackendXL
+		format.XL.Version = formatXLVersionV1
+		format.XL.Disk = jbod[index]
+		format.XL.JBOD = jbod
+		formatConfigs[index] = format
 	}
 	// Re order jbod for failure case.
 	var jbod1 = make([]string, 8)
@@ -317,7 +297,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 // a given disk to test healing a corrupted disk
 func TestFormatXLHealCorruptedDisks(t *testing.T) {
 	// Create an instance of xl backend.
-	obj, fsDirs, err := prepareXL()
+	obj, fsDirs, err := prepareXL16()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -391,7 +371,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 // some of format.json
 func TestFormatXLReorderByInspection(t *testing.T) {
 	// Create an instance of xl backend.
-	obj, fsDirs, err := prepareXL()
+	obj, fsDirs, err := prepareXL16()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -459,7 +439,7 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 //  - invalid JBOD
 //  - invalid Disk uuid
 func TestFormatXL(t *testing.T) {
-	formatInputCases := [][]*formatConfigV1{
+	formatInputCases := [][]*formatXLV1{
 		genFormatXLValid(),
 		genFormatXLInvalidVersion(),
 		genFormatXLInvalidFormat(),
@@ -470,7 +450,7 @@ func TestFormatXL(t *testing.T) {
 		genFormatXLInvalidDisksOrder(),
 	}
 	testCases := []struct {
-		formatConfigs []*formatConfigV1
+		formatConfigs []*formatXLV1
 		shouldPass    bool
 	}{
 		{
@@ -525,22 +505,20 @@ func TestSavedUUIDOrder(t *testing.T) {
 		shouldPass bool
 	}, 8)
 	jbod := make([]string, 8)
-	formatConfigs := make([]*formatConfigV1, 8)
+	formatConfigs := make([]*formatXLV1, 8)
 	for index := range jbod {
 		jbod[index] = mustGetUUID()
 		uuidTestCases[index].uuid = jbod[index]
 		uuidTestCases[index].shouldPass = true
 	}
 	for index := range jbod {
-		formatConfigs[index] = &formatConfigV1{
-			Version: formatFileV1,
-			Format:  formatBackendXL,
-			XL: &xlFormat{
-				Version: xlFormatBackendV1,
-				Disk:    jbod[index],
-				JBOD:    jbod,
-			},
-		}
+		format := &formatXLV1{}
+		format.Version = formatMetaVersionV1
+		format.Format = formatBackendXL
+		format.XL.Version = formatXLVersionV1
+		format.XL.Disk = jbod[index]
+		format.XL.JBOD = jbod
+		formatConfigs[index] = format
 	}
 	// Re order jbod for failure case.
 	var jbod1 = make([]string, 8)
@@ -653,172 +631,18 @@ func TestGenericFormatCheckXL(t *testing.T) {
 		t.Fatalf("Should fail here")
 	}
 	errs = []error{nil}
-	if err := genericFormatCheckXL([]*formatConfigV1{genFormatFS()}, errs); err == nil {
+	format := &formatXLV1{}
+	format.Version = formatMetaVersionV1
+	format.Format = formatBackendFS
+	if err := genericFormatCheckXL([]*formatXLV1{format}, errs); err == nil {
 		t.Fatalf("Should fail here")
 	}
 	errs = []error{errFaultyDisk}
-	if err := genericFormatCheckXL([]*formatConfigV1{genFormatFS()}, errs); err == nil {
+	if err := genericFormatCheckXL([]*formatXLV1{format}, errs); err == nil {
 		t.Fatalf("Should fail here")
 	}
 }
 
-// TestFSCheckFormatFSErr - test loadFormatFS loading older format.
-func TestFSCheckFormatFSErr(t *testing.T) {
-	// Prepare for testing
-	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer os.RemoveAll(disk)
-
-	// Assign a new UUID.
-	uuid := mustGetUUID()
-
-	// Initialize meta volume, if volume already exists ignores it.
-	if err := initMetaVolumeFS(disk, uuid); err != nil {
-		t.Fatal(err)
-	}
-
-	testCases := []struct {
-		format         *formatConfigV1
-		formatWriteErr error
-		formatCheckErr error
-		shouldPass     bool
-	}{
-		{
-			format: &formatConfigV1{
-				Version: formatFileV1,
-				Format:  formatBackendFS,
-				FS: &fsFormat{
-					Version: fsFormatBackendV1,
-				},
-			},
-			formatCheckErr: nil,
-			shouldPass:     true,
-		},
-		{
-			format: &formatConfigV1{
-				Version: formatFileV1,
-				Format:  formatBackendFS,
-				FS: &fsFormat{
-					Version: "10",
-				},
-			},
-			formatCheckErr: errors.New("Unknown backend FS format version '10'"),
-			shouldPass:     false,
-		},
-		{
-			format: &formatConfigV1{
-				Version: formatFileV1,
-				Format:  "garbage",
-				FS: &fsFormat{
-					Version: fsFormatBackendV1,
-				},
-			},
-			formatCheckErr: errors.New("FS backend format required. Found 'garbage'"),
-		},
-		{
-			format: &formatConfigV1{
-				Version: "-1",
-				Format:  formatBackendFS,
-				FS: &fsFormat{
-					Version: fsFormatBackendV1,
-				},
-			},
-			formatCheckErr: errors.New("Unknown format file version '-1'"),
-		},
-	}
-
-	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
-	for i, testCase := range testCases {
-		lk, err := lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
-		if err != nil {
-			t.Fatal(err)
-		}
-		_, err = testCase.format.WriteTo(lk)
-		lk.Close()
-		if err != nil {
-			t.Fatalf("Test %d: Expected nil, got %s", i+1, err)
-		}
-
-		lk, err = lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		formatCfg := &formatConfigV1{}
-		_, err = formatCfg.ReadFrom(lk)
-		lk.Close()
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = formatCfg.CheckFS()
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Should not fail with unexpected %s, expected nil", i+1, err)
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Should fail with expected %s, got nil", i+1, testCase.formatCheckErr)
-		}
-		if err != nil && !testCase.shouldPass {
-			if errors2.Cause(err).Error() != testCase.formatCheckErr.Error() {
-				t.Errorf("Test %d: Should fail with expected %s, got %s", i+1, testCase.formatCheckErr, err)
-			}
-		}
-	}
-}
-
-// TestFSCheckFormatFS - test loadFormatFS with healty and faulty disks
-func TestFSCheckFormatFS(t *testing.T) {
-	// Prepare for testing
-	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer os.RemoveAll(disk)
-
-	// Assign a new UUID.
-	uuid := mustGetUUID()
-
-	// Initialize meta volume, if volume already exists ignores it.
-	if err := initMetaVolumeFS(disk, uuid); err != nil {
-		t.Fatal(err)
-	}
-
-	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
-	lk, err := lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	format := newFSFormatV1()
-	_, err = format.WriteTo(lk)
-	lk.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Loading corrupted format file
-	file, err := os.OpenFile((fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
-	if err != nil {
-		t.Fatal("Should not fail here", err)
-	}
-	file.Write([]byte{'b'})
-	file.Close()
-
-	lk, err = lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	format = &formatConfigV1{}
-	_, err = format.ReadFrom(lk)
-	lk.Close()
-	if err == nil {
-		t.Fatal("Should return an error here")
-	}
-
-	// Loading format file from disk not found.
-	os.RemoveAll(disk)
-	_, err = lock.LockedOpenFile((fsFormatPath), os.O_RDONLY, 0600)
-	if err != nil && !os.IsNotExist(err) {
-		t.Fatal("Should return 'format.json' does not exist, but got", err)
-	}
-}
-
 func TestLoadFormatXLErrs(t *testing.T) {
 	nDisks := 16
 	fsDirs, err := getRandomDisks(nDisks)

cmd/fs-v1-helpers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"fmt"
 	"io"
 	"os"
 	pathutil "path"
@@ -91,24 +90,6 @@ func fsRemoveDir(dirPath string) (err error) {
 	return nil
 }
 
-// Creates a new directory, parent dir is also recursively created
-// if it doesn't exist.
-func fsMkdirAll(dirPath string) (err error) {
-	if dirPath == "" {
-		return errors.Trace(errInvalidArgument)
-	}
-
-	if err = checkPathLength(dirPath); err != nil {
-		return errors.Trace(err)
-	}
-
-	if err = os.MkdirAll(dirPath, 0777); err != nil {
-		return errors.Trace(err)
-	}
-
-	return nil
-}
-
 // Creates a new directory, parent dir should exist
 // otherwise returns an error. If directory already
 // exists returns an error. Windows long paths
@@ -277,7 +258,7 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
 		return 0, errors.Trace(err)
 	}
 
-	if err := os.MkdirAll(pathutil.Dir(filePath), 0777); err != nil {
+	if err := mkdirAll(pathutil.Dir(filePath), 0777); err != nil {
 		return 0, errors.Trace(err)
 	}
 
@@ -374,19 +355,16 @@ func fsRenameFile(sourcePath, destPath string) error {
 	if err := checkPathLength(destPath); err != nil {
 		return errors.Trace(err)
 	}
+
 	// Verify if source path exists.
-	if _, err := os.Stat((sourcePath)); err != nil {
+	if _, err := os.Stat(sourcePath); err != nil {
 		return osErrToFSFileErr(err)
 	}
-	if err := os.MkdirAll(pathutil.Dir(destPath), 0777); err != nil {
-		return errors.Trace(err)
-	}
-	if err := os.Rename((sourcePath), (destPath)); err != nil {
-		if isSysErrCrossDevice(err) {
-			return errors.Trace(fmt.Errorf("%s (%s)->(%s)", errCrossDeviceLink, sourcePath, destPath))
-		}
+
+	if err := renameAll(sourcePath, destPath); err != nil {
 		return errors.Trace(err)
 	}
+
 	return nil
 }
 

cmd/fs-v1-helpers_test.go

@@ -28,28 +28,6 @@ import (
 	"github.com/minio/minio/pkg/lock"
 )
 
-// Tests - fsMkdirAll()
-func TestFSMkdirAll(t *testing.T) {
-	// create posix test setup
-	_, path, err := newPosixTestSetup()
-	if err != nil {
-		t.Fatalf("Unable to create posix test setup, %s", err)
-	}
-	defer os.RemoveAll(path)
-
-	if err = fsMkdirAll(""); errors.Cause(err) != errInvalidArgument {
-		t.Fatal("Unexpected error", err)
-	}
-
-	if err = fsMkdirAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong {
-		t.Fatal("Unexpected error", err)
-	}
-
-	if err = fsMkdirAll(pathJoin(path, "success-vol", "success-object")); err != nil {
-		t.Fatal("Unexpected error", err)
-	}
-}
-
 func TestFSRenameFile(t *testing.T) {
 	// create posix test setup
 	_, path, err := newPosixTestSetup()

cmd/fs-v1-metadata.go

@@ -24,7 +24,6 @@ import (
 	pathutil "path"
 	"sort"
 	"strings"
-	"time"
 
 	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/lock"
@@ -122,7 +121,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.
-	objInfo.UserDefined = cleanMetaETag(m.Meta)
+	objInfo.UserDefined = cleanMetadata(m.Meta)
 
 	// Success..
 	return objInfo
@@ -269,97 +268,6 @@ func newFSMetaV1() (fsMeta fsMetaV1) {
 	return fsMeta
 }
 
-// Check if disk has already a valid format, holds a read lock and
-// upon success returns it to the caller to be closed.
-func checkLockedValidFormatFS(fsPath string) (*lock.RLockedFile, error) {
-	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
-
-	rlk, err := lock.RLockedOpenFile((fsFormatPath))
-	if err != nil {
-		if os.IsNotExist(err) {
-			// If format.json not found then
-			// its an unformatted disk.
-			return nil, errors.Trace(errUnformattedDisk)
-		}
-		return nil, errors.Trace(err)
-	}
-
-	var format = &formatConfigV1{}
-	if err = format.LoadFormat(rlk.LockedFile); err != nil {
-		rlk.Close()
-		return nil, err
-	}
-
-	// Check format FS.
-	if err = format.CheckFS(); err != nil {
-		rlk.Close()
-		return nil, err
-	}
-
-	//  Always return read lock here and should be closed by the caller.
-	return rlk, errors.Trace(err)
-}
-
-// Creates a new format.json if unformatted.
-func createFormatFS(fsPath string) error {
-	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
-
-	// Attempt a write lock on formatConfigFile `format.json`
-	// file stored in minioMetaBucket(.minio.sys) directory.
-	lk, err := lock.TryLockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
-	if err != nil {
-		return errors.Trace(err)
-	}
-	// Close the locked file upon return.
-	defer lk.Close()
-
-	// Load format on disk, checks if we are unformatted
-	// writes the new format.json
-	var format = &formatConfigV1{}
-	err = format.LoadFormat(lk)
-	if errors.Cause(err) == errUnformattedDisk {
-		_, err = newFSFormat().WriteTo(lk)
-		return err
-	}
-	return err
-}
-
-func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) {
-	// This loop validates format.json by holding a read lock and
-	// proceeds if disk unformatted to hold non-blocking WriteLock
-	// If for some reason non-blocking WriteLock fails and the error
-	// is lock.ErrAlreadyLocked i.e some other process is holding a
-	// lock we retry in the loop again.
-	for {
-		// Validate the `format.json` for expected values.
-		rlk, err = checkLockedValidFormatFS(fsPath)
-		switch {
-		case err == nil:
-			// Holding a read lock ensures that any write lock operation
-			// is blocked if attempted in-turn avoiding corruption on
-			// the backend disk.
-			return rlk, nil
-		case errors.Cause(err) == errUnformattedDisk:
-			if err = createFormatFS(fsPath); err != nil {
-				// Existing write locks detected.
-				if errors.Cause(err) == lock.ErrAlreadyLocked {
-					// Lock already present, sleep and attempt again.
-					time.Sleep(100 * time.Millisecond)
-					continue
-				}
-
-				// Unexpected error, return.
-				return nil, err
-			}
-
-			// Loop will continue to attempt a read-lock on `format.json`.
-		default:
-			// Unhandled error return.
-			return nil, err
-		}
-	}
-}
-
 // Return if the part info in uploadedParts and CompleteParts are same.
 func isPartsSame(uploadedParts []objectPartInfo, CompleteParts []CompletePart) bool {
 	if len(uploadedParts) != len(CompleteParts) {

cmd/fs-v1-multipart.go

@@ -109,7 +109,7 @@ func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarke
 
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
-	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucketName, objectName))
+	objectMPartPathLock := fs.nsMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucketName, objectName))
 	if err := objectMPartPathLock.GetRLock(globalListingTimeout); err != nil {
 		return nil, false, errors.Trace(err)
 	}
@@ -408,7 +408,7 @@ func (fs fsObjects) NewMultipartUpload(bucket, object string, meta map[string]st
 
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
-	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
+	objectMPartPathLock := fs.nsMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
 	if err := objectMPartPathLock.GetLock(globalOperationTimeout); err != nil {
 		return "", err
 	}
@@ -437,17 +437,34 @@ func partToAppend(fsMeta fsMetaV1, fsAppendMeta fsMetaV1) (part objectPartInfo,
 // object. Internally incoming data is written to '.minio.sys/tmp' location
 // and safely renamed to '.minio.sys/multipart' for reach parts.
 func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int,
-	startOffset int64, length int64, metadata map[string]string) (pi PartInfo, e error) {
+	startOffset int64, length int64, metadata map[string]string, srcEtag string) (pi PartInfo, e error) {
+
+	// Hold read locks on source object only if we are
+	// going to read data from source object.
+	objectSRLock := fs.nsMutex.NewNSLock(srcBucket, srcObject)
+	if err := objectSRLock.GetRLock(globalObjectTimeout); err != nil {
+		return pi, err
+	}
+	defer objectSRLock.RUnlock()
 
 	if err := checkNewMultipartArgs(srcBucket, srcObject, fs); err != nil {
 		return pi, err
 	}
 
+	if srcEtag != "" {
+		etag, err := fs.getObjectETag(srcBucket, srcObject)
+		if err != nil {
+			return pi, toObjectErr(err, srcBucket, srcObject)
+		}
+		if etag != srcEtag {
+			return pi, toObjectErr(errors.Trace(InvalidETag{}), srcBucket, srcObject)
+		}
+	}
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
 
 	go func() {
-		if gerr := fs.GetObject(srcBucket, srcObject, startOffset, length, pipeWriter); gerr != nil {
+		if gerr := fs.getObject(srcBucket, srcObject, startOffset, length, pipeWriter, srcEtag); gerr != nil {
 			errorIf(gerr, "Unable to read %s/%s.", srcBucket, srcObject)
 			pipeWriter.CloseWithError(gerr)
 			return
@@ -491,7 +508,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
 
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
-	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
+	objectMPartPathLock := fs.nsMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
 	if err := objectMPartPathLock.GetLock(globalOperationTimeout); err != nil {
 		return pi, err
 	}
@@ -557,7 +574,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
 	partPath := pathJoin(bucket, object, uploadID, partSuffix)
 	// Lock the part so that another part upload with same part-number gets blocked
 	// while the part is getting appended in the background.
-	partLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, partPath)
+	partLock := fs.nsMutex.NewNSLock(minioMetaMultipartBucket, partPath)
 	if err = partLock.GetLock(globalOperationTimeout); err != nil {
 		return pi, err
 	}
@@ -689,7 +706,7 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
 
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
-	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
+	objectMPartPathLock := fs.nsMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
 	if err := objectMPartPathLock.GetRLock(globalListingTimeout); err != nil {
 		return lpi, errors.Trace(err)
 	}
@@ -714,6 +731,12 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		return oi, err
 	}
 
+	// Hold write lock on the object.
+	destLock := fs.nsMutex.NewNSLock(bucket, object)
+	if err := destLock.GetLock(globalObjectTimeout); err != nil {
+		return oi, err
+	}
+	defer destLock.Unlock()
 	// Check if an object is present as one of the parent dir.
 	if fs.parentDirIsObject(bucket, pathutil.Dir(object)) {
 		return oi, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object)
@@ -734,7 +757,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
-	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
+	objectMPartPathLock := fs.nsMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
 	if err = objectMPartPathLock.GetLock(globalOperationTimeout); err != nil {
 		return oi, err
 	}
@@ -974,7 +997,7 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
-	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket,
+	objectMPartPathLock := fs.nsMutex.NewNSLock(minioMetaMultipartBucket,
 		pathJoin(bucket, object))
 	if err := objectMPartPathLock.GetLock(globalOperationTimeout); err != nil {
 		return err

cmd/fs-v1.go

@@ -26,6 +26,7 @@ import (
 	"path/filepath"
 	"sort"
 	"syscall"
+	"time"
 
 	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
@@ -52,6 +53,9 @@ type fsObjects struct {
 
 	// To manage the appendRoutine go0routines
 	bgAppend *backgroundAppend
+
+	// name space mutex for object layer
+	nsMutex *nsLockMap
 }
 
 // Initializes meta volume on all the fs path.
@@ -138,6 +142,7 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		bgAppend: &backgroundAppend{
 			infoMap: make(map[string]bgAppendPartsInfo),
 		},
+		nsMutex: newNSLock(false),
 	}
 
 	// Once the filesystem has initialized hold the read lock for
@@ -183,6 +188,18 @@ func (fs fsObjects) StorageInfo() StorageInfo {
 	return storageInfo
 }
 
+// Locking operations
+
+// List namespace locks held in object layer
+func (fs fsObjects) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
+	return []VolumeLockInfo{}, NotImplemented{}
+}
+
+// Clear namespace locks held in object layer
+func (fs fsObjects) ClearLocks([]VolumeLockInfo) error {
+	return NotImplemented{}
+}
+
 /// Bucket operations
 
 // getBucketDir - will convert incoming bucket names to
@@ -213,6 +230,11 @@ func (fs fsObjects) statBucketDir(bucket string) (os.FileInfo, error) {
 // MakeBucket - create a new bucket, returns if it
 // already exists.
 func (fs fsObjects) MakeBucketWithLocation(bucket, location string) error {
+	bucketLock := fs.nsMutex.NewNSLock(bucket, "")
+	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
+		return err
+	}
+	defer bucketLock.Unlock()
 	bucketDir, err := fs.getBucketDir(bucket)
 	if err != nil {
 		return toObjectErr(err, bucket)
@@ -313,7 +335,29 @@ func (fs fsObjects) DeleteBucket(bucket string) error {
 // CopyObject - copy object source object to destination object.
 // if source object and destination object are same we only
 // update metadata.
-func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string) (oi ObjectInfo, e error) {
+func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string, srcEtag string) (oi ObjectInfo, e error) {
+	cpSrcDstSame := srcBucket == dstBucket && srcObject == dstObject
+	// Hold write lock on destination since in both cases
+	// - if source and destination are same
+	// - if source and destination are different
+	// it is the sole mutating state.
+	objectDWLock := fs.nsMutex.NewNSLock(dstBucket, dstObject)
+	if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
+		return oi, err
+	}
+	defer objectDWLock.Unlock()
+	// if source and destination are different, we have to hold
+	// additional read lock as well to protect against writes on
+	// source.
+	if !cpSrcDstSame {
+		// Hold read locks on source object only if we are
+		// going to read data from source object.
+		objectSRLock := fs.nsMutex.NewNSLock(srcBucket, srcObject)
+		if err := objectSRLock.GetRLock(globalObjectTimeout); err != nil {
+			return oi, err
+		}
+		defer objectSRLock.RUnlock()
+	}
 	if _, err := fs.statBucketDir(srcBucket); err != nil {
 		return oi, toObjectErr(err, srcBucket)
 	}
@@ -323,6 +367,15 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 	if err != nil {
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
+	if srcEtag != "" {
+		etag, perr := fs.getObjectETag(srcBucket, srcObject)
+		if perr != nil {
+			return oi, toObjectErr(perr, srcBucket, srcObject)
+		}
+		if etag != srcEtag {
+			return oi, toObjectErr(errors.Trace(InvalidETag{}), srcBucket, srcObject)
+		}
+	}
 
 	// Check if this request is only metadata update.
 	cpMetadataOnly := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
@@ -355,7 +408,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 
 	go func() {
 		var startOffset int64 // Read the whole file.
-		if gerr := fs.GetObject(srcBucket, srcObject, startOffset, length, pipeWriter); gerr != nil {
+		if gerr := fs.getObject(srcBucket, srcObject, startOffset, length, pipeWriter, ""); gerr != nil {
 			errorIf(gerr, "Unable to read %s/%s.", srcBucket, srcObject)
 			pipeWriter.CloseWithError(gerr)
 			return
@@ -368,7 +421,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 		return oi, toObjectErr(err, dstBucket, dstObject)
 	}
 
-	objInfo, err := fs.PutObject(dstBucket, dstObject, hashReader, metadata)
+	objInfo, err := fs.putObject(dstBucket, dstObject, hashReader, metadata)
 	if err != nil {
 		return oi, toObjectErr(err, dstBucket, dstObject)
 	}
@@ -385,11 +438,22 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer) (err error) {
+func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) {
 	if err = checkBucketAndObjectNamesFS(bucket, object); err != nil {
 		return err
 	}
 
+	// Lock the object before reading.
+	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
+		return err
+	}
+	defer objectLock.RUnlock()
+	return fs.getObject(bucket, object, offset, length, writer, etag)
+}
+
+// getObject - wrapper for GetObject
+func (fs fsObjects) getObject(bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) {
 	if _, err = fs.statBucketDir(bucket); err != nil {
 		return toObjectErr(err, bucket)
 	}
@@ -419,6 +483,15 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
 		defer fs.rwPool.Close(fsMetaPath)
 	}
 
+	if etag != "" {
+		objEtag, perr := fs.getObjectETag(bucket, object)
+		if perr != nil {
+			return toObjectErr(errors.Trace(perr), bucket, object)
+		}
+		if objEtag != etag {
+			return toObjectErr(errors.Trace(InvalidETag{}), bucket, object)
+		}
+	}
 	// Read the object, doesn't exist returns an s3 compatible error.
 	fsObjPath := pathJoin(fs.fsPath, bucket, object)
 	reader, size, err := fsOpenFile(fsObjPath, offset)
@@ -516,6 +589,13 @@ func checkBucketAndObjectNamesFS(bucket, object string) error {
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (fs fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
+	// Lock the object before reading.
+	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
+		return oi, err
+	}
+	defer objectLock.RUnlock()
+
 	if err := checkBucketAndObjectNamesFS(bucket, object); err != nil {
 		return oi, err
 	}
@@ -552,11 +632,21 @@ func (fs fsObjects) parentDirIsObject(bucket, parent string) bool {
 // Additionally writes `fs.json` which carries the necessary metadata
 // for future object operations.
 func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
+	// Lock the object.
+	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	if err := objectLock.GetLock(globalObjectTimeout); err != nil {
+		return objInfo, err
+	}
+	defer objectLock.Unlock()
+	return fs.putObject(bucket, object, data, metadata)
+}
+
+// putObject - wrapper for PutObject
+func (fs fsObjects) putObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
 	// No metadata is set, allocate a new one.
 	if metadata == nil {
 		metadata = make(map[string]string)
 	}
-
 	var err error
 
 	// Validate if bucket name is valid and exists.
@@ -575,7 +665,7 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
 		if fs.parentDirIsObject(bucket, path.Dir(object)) {
 			return ObjectInfo{}, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object)
 		}
-		if err = fsMkdirAll(pathJoin(fs.fsPath, bucket, object)); err != nil {
+		if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 		var fi os.FileInfo
@@ -634,7 +724,6 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
 	bytesWritten, err := fsCreateFile(fsTmpObjPath, data, buf, data.Size())
 	if err != nil {
 		fsRemoveFile(fsTmpObjPath)
-		errorIf(err, "Failed to create object %s/%s", bucket, object)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
@@ -678,6 +767,13 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
 // DeleteObject - deletes an object from a bucket, this operation is destructive
 // and there are no rollbacks supported.
 func (fs fsObjects) DeleteObject(bucket, object string) error {
+	// Acquire a write lock before deleting the object.
+	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	if err := objectLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
+	defer objectLock.Unlock()
+
 	if err := checkBucketAndObjectNamesFS(bucket, object); err != nil {
 		return err
 	}
@@ -826,7 +922,7 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 	// Convert entry to ObjectInfo
 	entryToObjectInfo := func(entry string) (objInfo ObjectInfo, err error) {
 		// Protect the entry from concurrent deletes, or renames.
-		objectLock := globalNSMutex.NewNSLock(bucket, entry)
+		objectLock := fs.nsMutex.NewNSLock(bucket, entry)
 		if err = objectLock.GetRLock(globalListingTimeout); err != nil {
 			return ObjectInfo{}, err
 		}
@@ -961,8 +1057,3 @@ func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, ma
 func (fs fsObjects) ListBucketsHeal() ([]BucketInfo, error) {
 	return []BucketInfo{}, errors.Trace(NotImplemented{})
 }
-
-func (fs fsObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
-	delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
-	return lmi, errors.Trace(NotImplemented{})
-}

cmd/gateway-handlers.go

@@ -81,7 +81,6 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 	objInfo, err := getObjectInfo(bucket, object)
 	if err != nil {
-		errorIf(err, "Unable to fetch object info.")
 		apiErr := toAPIErrorCode(err)
 		if apiErr == ErrNoSuchKey {
 			apiErr = errAllowableObjectNotFound(bucket, r)
@@ -129,7 +128,7 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
 	setHeadGetRespHeaders(w, r.URL.Query())
 	httpWriter := ioutil.WriteOnClose(w)
 	// Reads the object at startOffset and writes to mw.
-	if err = getObject(bucket, object, startOffset, length, httpWriter); err != nil {
+	if err = getObject(bucket, object, startOffset, length, httpWriter, objInfo.ETag); err != nil {
 		errorIf(err, "Unable to write to client.")
 		if !httpWriter.HasWritten() {
 			// Error response only if no data has been written to client yet. i.e if
@@ -199,7 +198,6 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
 	// Get Content-Md5 sent by client and verify if valid
 	md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5"))
 	if err != nil {
-		errorIf(err, "Unable to validate content-md5 format.")
 		writeErrorResponse(w, ErrInvalidDigest, r.URL)
 		return
 	}
@@ -211,7 +209,6 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
 		sizeStr := r.Header.Get("x-amz-decoded-content-length")
 		size, err = strconv.ParseInt(sizeStr, 10, 64)
 		if err != nil {
-			errorIf(err, "Unable to parse `x-amz-decoded-content-length` into its integer value", sizeStr)
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 			return
 		}
@@ -230,7 +227,6 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
 	// Extract metadata to be saved from incoming HTTP header.
 	metadata, err := extractMetadataFromHeader(r.Header)
 	if err != nil {
-		errorIf(err, "found invalid http request header")
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
@@ -254,14 +250,6 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 	}
 
-	// Lock the object.
-	objectLock := globalNSMutex.NewNSLock(bucket, object)
-	if objectLock.GetLock(globalOperationTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
-	defer objectLock.Unlock()
-
 	var (
 		// Make sure we hex encode md5sum here.
 		md5hex    = hex.EncodeToString(md5Bytes)
@@ -385,7 +373,6 @@ func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.R
 	}
 	objInfo, err := getObjectInfo(bucket, object)
 	if err != nil {
-		errorIf(err, "Unable to fetch object info.")
 		apiErr := toAPIErrorCode(err)
 		if apiErr == ErrNoSuchKey {
 			apiErr = errAllowableObjectNotFound(bucket, r)
@@ -448,7 +435,6 @@ func (api gatewayAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *h
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -511,7 +497,6 @@ func (api gatewayAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -545,21 +530,18 @@ func (api gatewayAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *h
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	bp, err := objAPI.GetBucketPolicies(bucket)
 	if err != nil {
-		errorIf(err, "Unable to read bucket policy.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	policyBytes, err := json.Marshal(bp)
 	if err != nil {
-		errorIf(err, "Unable to read bucket policy.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -631,7 +613,6 @@ func (api gatewayAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Re
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(bucket, location)
 	if err != nil {
-		errorIf(err, "Unable to create a bucket.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -661,7 +642,6 @@ func (api gatewayAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http
 
 	// Attempt to delete bucket.
 	if err := objectAPI.DeleteBucket(bucket); err != nil {
-		errorIf(err, "Unable to delete a bucket.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -733,7 +713,6 @@ func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *htt
 	// marshalled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		errorIf(err, "Unable to list objects.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -813,7 +792,6 @@ func (api gatewayAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *htt
 	// serialized as XML and sent as S3 compatible response body.
 	listObjectsV2Info, err := listObjectsV2(bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
-		errorIf(err, "Unable to list objects. Args to listObjectsV2 are bucket=%s, prefix=%s, token=%s, delimiter=%s", bucket, prefix, token, delimiter)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -871,7 +849,6 @@ func (api gatewayAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.R
 	}
 
 	if _, err := getBucketInfo(bucket); err != nil {
-		errorIf(err, "Unable to fetch bucket info.")
 		writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
 		return
 	}
@@ -927,7 +904,6 @@ func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r
 	}
 
 	if _, err := getBucketInfo(bucket); err != nil {
-		errorIf(err, "Unable to fetch bucket info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}

cmd/gateway-main.go

@@ -112,6 +112,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
 	}
 
+	// Get "json" flag from command line argument and
+	// enable json and quite modes if jason flag is turned on.
+	jsonFlag := ctx.Bool("json") || ctx.GlobalBool("json")
+	if jsonFlag {
+		log.EnableJSON()
+	}
+
 	// Get quiet flag from command line argument.
 	quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
 	if quietFlag {

cmd/gateway-router.go

@@ -45,7 +45,7 @@ type Gateway interface {
 type GatewayLayer interface {
 	ObjectLayer
 
-	AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
+	AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error)
 	AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
 
 	AnonPutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (ObjectInfo, error)

cmd/gateway-startup-msg.go

@@ -53,6 +53,8 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
 	log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
 
-	log.Println(colorBlue("\nBrowser Access:"))
-	log.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
+	if globalIsBrowserEnabled {
+		log.Println(colorBlue("\nBrowser Access:"))
+		log.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
+	}
 }

cmd/gateway-unsupported.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"io"
+	"time"
 
 	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/pkg/errors"
@@ -38,7 +39,7 @@ func (a GatewayUnsupported) NewMultipartUpload(bucket string, object string, met
 }
 
 // CopyObjectPart copy part of object to other bucket and object
-func (a GatewayUnsupported) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string, uploadID string, partID int, startOffset int64, length int64, metadata map[string]string) (pi PartInfo, err error) {
+func (a GatewayUnsupported) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string, uploadID string, partID int, startOffset int64, length int64, metadata map[string]string, srcEtag string) (pi PartInfo, err error) {
 	return pi, errors.Trace(NotImplemented{})
 }
 
@@ -102,12 +103,6 @@ func (a GatewayUnsupported) ListObjectsHeal(bucket, prefix, marker, delimiter st
 	return loi, errors.Trace(NotImplemented{})
 }
 
-// ListUploadsHeal - Not implemented stub
-func (a GatewayUnsupported) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
-	delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
-	return lmi, errors.Trace(NotImplemented{})
-}
-
 // AnonListObjects - List objects anonymously
 func (a GatewayUnsupported) AnonListObjects(bucket string, prefix string, marker string, delimiter string,
 	maxKeys int) (loi ListObjectsInfo, err error) {
@@ -132,7 +127,7 @@ func (a GatewayUnsupported) AnonPutObject(bucket, object string, data *hash.Read
 }
 
 // AnonGetObject downloads object anonymously.
-func (a GatewayUnsupported) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
+func (a GatewayUnsupported) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
 	return errors.Trace(NotImplemented{})
 }
 
@@ -143,6 +138,18 @@ func (a GatewayUnsupported) AnonGetObjectInfo(bucket, object string) (objInfo Ob
 
 // CopyObject copies a blob from source container to destination container.
 func (a GatewayUnsupported) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string,
-	metadata map[string]string) (objInfo ObjectInfo, err error) {
+	metadata map[string]string, srcEtag string) (objInfo ObjectInfo, err error) {
 	return objInfo, errors.Trace(NotImplemented{})
 }
+
+// Locking operations
+
+// ListLocks lists namespace locks held in object layer
+func (a GatewayUnsupported) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
+	return []VolumeLockInfo{}, errors.Trace(NotImplemented{})
+}
+
+// ClearLocks clears namespace locks held in object layer
+func (a GatewayUnsupported) ClearLocks([]VolumeLockInfo) error {
+	return errors.Trace(NotImplemented{})
+}

cmd/gateway/azure/gateway-azure-anonymous.go

@@ -145,7 +145,7 @@ func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo minio.Bucket
 
 // AnonGetObject - SendGET request without authentication.
 // This is needed when clients send GET requests on objects that can be downloaded without auth.
-func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
+func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
 	h := make(http.Header)
 	if length > 0 && startOffset > 0 {
 		h.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))

cmd/gateway/azure/gateway-azure.go

@@ -19,7 +19,6 @@ package azure
 import (
 	"bytes"
 	"crypto/rand"
-	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
@@ -38,6 +37,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/hash"
+	sha256 "github.com/minio/sha256-simd"
 
 	minio "github.com/minio/minio/cmd"
 )
@@ -554,7 +554,7 @@ func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken, delimite
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (a *azureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
+func (a *azureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
 	// startOffset cannot be negative.
 	if startOffset < 0 {
 		return azureToObjectError(errors.Trace(minio.InvalidRange{}), bucket, object)
@@ -621,7 +621,7 @@ func (a *azureObjects) PutObject(bucket, object string, data *hash.Reader, metad
 
 // CopyObject - Copies a blob from source container to destination container.
 // Uses Azure equivalent CopyBlob API.
-func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string, srcEtag string) (objInfo minio.ObjectInfo, err error) {
 	srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL()
 	destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject)
 	azureMeta, props, err := s3MetaToAzureProperties(metadata)

cmd/gateway/b2/gateway-b2-anonymous.go

@@ -43,7 +43,7 @@ func mkRange(offset, size int64) string {
 
 // AnonGetObject - performs a plain http GET request on a public resource,
 // fails if the resource is not public.
-func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
+func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
 	uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
 	req, err := http.NewRequest("GET", uri, nil)
 	if err != nil {

cmd/gateway/b2/gateway-b2.go

@@ -379,7 +379,7 @@ func (l *b2Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter s
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *b2Objects) GetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
+func (l *b2Objects) GetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
 	bkt, err := l.Bucket(bucket)
 	if err != nil {
 		return err
@@ -523,7 +523,7 @@ func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, met
 
 // CopyObject copies a blob from source container to destination container.
 func (l *b2Objects) CopyObject(srcBucket string, srcObject string, dstBucket string,
-	dstObject string, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+	dstObject string, metadata map[string]string, srcEtag string) (objInfo minio.ObjectInfo, err error) {
 	return objInfo, errors.Trace(minio.NotImplemented{})
 }
 

cmd/gateway/gcs/gateway-gcs-anonymous.go

@@ -33,7 +33,7 @@ func toGCSPublicURL(bucket, object string) string {
 }
 
 // AnonGetObject - Get object anonymously
-func (l *gcsGateway) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
+func (l *gcsGateway) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
 	req, err := http.NewRequest("GET", toGCSPublicURL(bucket, object), nil)
 	if err != nil {
 		return gcsToObjectError(errors.Trace(err), bucket, object)

cmd/gateway/gcs/gateway-gcs.go

@@ -725,7 +725,7 @@ func (l *gcsGateway) ListObjectsV2(bucket, prefix, continuationToken, delimiter
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *gcsGateway) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
+func (l *gcsGateway) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error {
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
 	// otherwise gcs will just return object not exist in case of non-existing bucket
 	if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
@@ -813,7 +813,7 @@ func (l *gcsGateway) PutObject(bucket string, key string, data *hash.Reader, met
 
 // CopyObject - Copies a blob from source container to destination container.
 func (l *gcsGateway) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string,
-	metadata map[string]string) (minio.ObjectInfo, error) {
+	metadata map[string]string, srcEtag string) (minio.ObjectInfo, error) {
 
 	src := l.client.Bucket(srcBucket).Object(srcObject)
 	dst := l.client.Bucket(destBucket).Object(destObject)

cmd/gateway/manta/gateway-manta.go

@@ -27,12 +27,10 @@ import (
 	"path"
 	"strings"
 
-	"github.com/hashicorp/errwrap"
 	triton "github.com/joyent/triton-go"
 	"github.com/joyent/triton-go/authentication"
-	tclient "github.com/joyent/triton-go/client"
+	terrors "github.com/joyent/triton-go/errors"
 	"github.com/joyent/triton-go/storage"
-
 	"github.com/minio/cli"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/pkg/auth"
@@ -68,6 +66,7 @@ ENVIRONMENT VARIABLES:
      MINIO_ACCESS_KEY: The Manta account name.
      MINIO_SECRET_KEY: A KeyID associated with the Manta account.
      MANTA_KEY_MATERIAL: The path to the SSH Key associated with the Manta account if the MINIO_SECRET_KEY is not in SSH Agent.
+	 MANTA_SUBUSER: The username of a user who has limited access to your account.
 
   BROWSER:
      MINIO_BROWSER: To disable web browser access, set this value to "off".
@@ -140,7 +139,14 @@ func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.GatewayLayer, err
 	keyMaterial := os.Getenv("MANTA_KEY_MATERIAL")
 
 	if keyMaterial == "" {
-		signer, err = authentication.NewSSHAgentSigner(creds.SecretKey, creds.AccessKey)
+		input := authentication.SSHAgentSignerInput{
+			KeyID:       creds.SecretKey,
+			AccountName: creds.AccessKey,
+		}
+		if userName, ok := os.LookupEnv("MANTA_SUBUSER"); ok {
+			input.Username = userName
+		}
+		signer, err = authentication.NewSSHAgentSigner(input)
 		if err != nil {
 			return nil, errors.Trace(err)
 		}
@@ -168,7 +174,16 @@ func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.GatewayLayer, err
 			keyBytes = []byte(keyMaterial)
 		}
 
-		signer, err = authentication.NewPrivateKeySigner(creds.SecretKey, keyBytes, creds.AccessKey)
+		input := authentication.PrivateKeySignerInput{
+			KeyID:              creds.SecretKey,
+			PrivateKeyMaterial: keyBytes,
+			AccountName:        creds.AccessKey,
+		}
+		if userName, ok := os.LookupEnv("MANTA_SUBUSER"); ok {
+			input.Username = userName
+		}
+
+		signer, err = authentication.NewPrivateKeySigner(input)
 		if err != nil {
 			return nil, errors.Trace(err)
 		}
@@ -324,7 +339,7 @@ func (t *tritonObjects) ListObjects(bucket, prefix, marker, delimiter string, ma
 	}
 	objs, err = t.client.Dir().List(ctx, input)
 	if err != nil {
-		if tclient.IsResourceNotFoundError(err) {
+		if terrors.IsResourceNotFoundError(err) {
 			return result, nil
 		}
 		return result, errors.Trace(err)
@@ -401,7 +416,7 @@ func (t *tritonObjects) ListObjectsV2(bucket, prefix, continuationToken, delimit
 	}
 	objs, err = t.client.Dir().List(ctx, input)
 	if err != nil {
-		if tclient.IsResourceNotFoundError(err) {
+		if terrors.IsResourceNotFoundError(err) {
 			return result, nil
 		}
 		return result, errors.Trace(err)
@@ -452,7 +467,7 @@ func (t *tritonObjects) ListObjectsV2(bucket, prefix, continuationToken, delimit
 // indicates the total length of the object.
 //
 // https://apidocs.joyent.com/manta/api.html#GetObject
-func (t *tritonObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
+func (t *tritonObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
 	// Start offset cannot be negative.
 	if startOffset < 0 {
 		return errors.Trace(fmt.Errorf("Unexpected error"))
@@ -490,14 +505,10 @@ func (t *tritonObjects) GetObjectInfo(bucket, object string) (objInfo minio.Obje
 		ObjectPath: path.Join(mantaRoot, bucket, object),
 	})
 	if err != nil {
-		errType := &tclient.MantaError{}
-		if errwrap.ContainsType(err, errType) {
-			mantaErr := errwrap.GetType(err, errType).(*tclient.MantaError)
-			if mantaErr.StatusCode == http.StatusNotFound {
-				return objInfo, minio.ObjectNotFound{
-					Bucket: bucket,
-					Object: object,
-				}
+		if terrors.IsStatusNotFoundCode(err) {
+			return objInfo, minio.ObjectNotFound{
+				Bucket: bucket,
+				Object: object,
 			}
 		}
 
@@ -552,7 +563,7 @@ func (t *tritonObjects) PutObject(bucket, object string, data *hash.Reader, meta
 // Uses Manta Snaplinks API.
 //
 // https://apidocs.joyent.com/manta/api.html#PutSnapLink
-func (t *tritonObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (t *tritonObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string, srcEtag string) (objInfo minio.ObjectInfo, err error) {
 	ctx := context.Background()
 	if err = t.client.SnapLinks().Put(ctx, &storage.PutSnapLinkInput{
 		SourcePath: path.Join(mantaRoot, srcBucket, srcObject),

cmd/gateway/oss/gateway-oss-anonymous.go

@@ -29,8 +29,8 @@ func (l *ossObjects) AnonPutObject(bucket, object string, data *hash.Reader, met
 }
 
 // AnonGetObject - Get object anonymously
-func (l *ossObjects) AnonGetObject(bucket, key string, startOffset, length int64, writer io.Writer) error {
-	return ossGetObject(l.anonClient, bucket, key, startOffset, length, writer)
+func (l *ossObjects) AnonGetObject(bucket, key string, startOffset, length int64, writer io.Writer, etag string) error {
+	return ossGetObject(l.anonClient, bucket, key, startOffset, length, writer, etag)
 }
 
 // AnonGetObjectInfo - Get object info anonymously

cmd/gateway/oss/gateway-oss.go

@@ -506,7 +506,7 @@ func (l *ossObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func ossGetObject(client *oss.Client, bucket, key string, startOffset, length int64, writer io.Writer) error {
+func ossGetObject(client *oss.Client, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error {
 	if length < 0 && length != -1 {
 		return ossToObjectError(errors.Trace(fmt.Errorf("Invalid argument")), bucket, key)
 	}
@@ -539,8 +539,8 @@ func ossGetObject(client *oss.Client, bucket, key string, startOffset, length in
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *ossObjects) GetObject(bucket, key string, startOffset, length int64, writer io.Writer) error {
-	return ossGetObject(l.Client, bucket, key, startOffset, length, writer)
+func (l *ossObjects) GetObject(bucket, key string, startOffset, length int64, writer io.Writer, etag string) error {
+	return ossGetObject(l.Client, bucket, key, startOffset, length, writer, etag)
 }
 
 func translatePlainError(err error) error {
@@ -618,7 +618,7 @@ func (l *ossObjects) PutObject(bucket, object string, data *hash.Reader, metadat
 }
 
 // CopyObject copies an object from source bucket to a destination bucket.
-func (l *ossObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (l *ossObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string, srcEtag string) (objInfo minio.ObjectInfo, err error) {
 	bkt, err := l.Client.Bucket(srcBucket)
 	if err != nil {
 		return objInfo, ossToObjectError(errors.Trace(err), srcBucket, srcObject)
@@ -804,7 +804,7 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par
 // CopyObjectPart creates a part in a multipart upload by copying
 // existing object or a part of it.
 func (l *ossObjects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject, uploadID string,
-	partID int, startOffset, length int64, metadata map[string]string) (p minio.PartInfo, err error) {
+	partID int, startOffset, length int64, metadata map[string]string, srcEtag string) (p minio.PartInfo, err error) {
 
 	bkt, err := l.Client.Bucket(destBucket)
 	if err != nil {

cmd/gateway/s3/gateway-s3-anonymous.go

@@ -37,7 +37,7 @@ func (l *s3Objects) AnonPutObject(bucket string, object string, data *hash.Reade
 }
 
 // AnonGetObject - Get object anonymously
-func (l *s3Objects) AnonGetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
+func (l *s3Objects) AnonGetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error {
 	opts := miniogo.GetObjectOptions{}
 	if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
 		return minio.ErrorRespToObjectError(errors.Trace(err), bucket, key)

cmd/gateway/s3/gateway-s3.go

@@ -254,7 +254,7 @@ func (l *s3Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter s
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *s3Objects) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
+func (l *s3Objects) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error {
 	if length < 0 && length != -1 {
 		return minio.ErrorRespToObjectError(errors.Trace(minio.InvalidRange{}), bucket, key)
 	}
@@ -298,7 +298,7 @@ func (l *s3Objects) PutObject(bucket string, object string, data *hash.Reader, m
 }
 
 // CopyObject copies an object from source bucket to a destination bucket.
-func (l *s3Objects) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (l *s3Objects) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string, metadata map[string]string, srcEtag string) (objInfo minio.ObjectInfo, err error) {
 	// Set this header such that following CopyObject() always sets the right metadata on the destination.
 	// metadata input is already a trickled down value from interpreting x-amz-metadata-directive at
 	// handler layer. So what we have right now is supposed to be applied on the destination object anyways.
@@ -354,7 +354,7 @@ func (l *s3Objects) PutObjectPart(bucket string, object string, uploadID string,
 // CopyObjectPart creates a part in a multipart upload by copying
 // existing object or a part of it.
 func (l *s3Objects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject, uploadID string,
-	partID int, startOffset, length int64, metadata map[string]string) (p minio.PartInfo, err error) {
+	partID int, startOffset, length int64, metadata map[string]string, srcEtag string) (p minio.PartInfo, err error) {
 
 	completePart, err := l.Client.CopyObjectPart(srcBucket, srcObject, destBucket, destObject,
 		uploadID, partID, startOffset, length, metadata)

cmd/gateway/sia/gateway-sia.go

@@ -410,7 +410,7 @@ func (s *siaObjects) ListObjects(bucket string, prefix string, marker string, de
 	return loi, nil
 }
 
-func (s *siaObjects) GetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
+func (s *siaObjects) GetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
 	dstFile := path.Join(s.TempDir, minio.MustGetUUID())
 	defer os.Remove(dstFile)
 

cmd/generic-handlers.go

@@ -229,7 +229,7 @@ func setBrowserCacheControlHandler(h http.Handler) http.Handler {
 }
 
 func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method == httpGET && guessIsBrowserReq(r) && globalIsBrowserEnabled {
+	if r.Method == http.MethodGet && guessIsBrowserReq(r) && globalIsBrowserEnabled {
 		// For all browser requests set appropriate Cache-Control policies
 		if hasPrefix(r.URL.Path, minioReservedBucketPath+"/") {
 			if hasSuffix(r.URL.Path, ".js") || r.URL.Path == minioReservedBucketPath+"/favicon.ico" {
@@ -341,24 +341,14 @@ type resourceHandler struct {
 	handler http.Handler
 }
 
-// List of http methods.
-const (
-	httpGET     = "GET"
-	httpPUT     = "PUT"
-	httpHEAD    = "HEAD"
-	httpPOST    = "POST"
-	httpDELETE  = "DELETE"
-	httpOPTIONS = "OPTIONS"
-)
-
 // List of default allowable HTTP methods.
 var defaultAllowableHTTPMethods = []string{
-	httpGET,
-	httpPUT,
-	httpHEAD,
-	httpPOST,
-	httpDELETE,
-	httpOPTIONS,
+	http.MethodGet,
+	http.MethodPut,
+	http.MethodHead,
+	http.MethodPost,
+	http.MethodDelete,
+	http.MethodOptions,
 }
 
 // setCorsHandler handler for CORS (Cross Origin Resource Sharing)

cmd/globals.go

@@ -123,8 +123,8 @@ var (
 	globalHTTPServerErrorCh = make(chan error)
 	globalOSSignalCh        = make(chan os.Signal, 1)
 
-	// Enable HTTP request/response headers and body logging.
-	globalHTTPTrace bool
+	// File to log HTTP request/response headers and body.
+	globalHTTPTraceFile *os.File
 
 	// List of admin peers.
 	globalAdminPeers = adminPeers{}
@@ -174,6 +174,7 @@ var (
 	colorBold   = color.New(color.Bold).SprintFunc()
 	colorBlue   = color.New(color.FgBlue).SprintfFunc()
 	colorYellow = color.New(color.FgYellow).SprintfFunc()
+	colorRed    = color.New(color.FgRed).SprintfFunc()
 )
 
 // Returns minio global information, as a key value map.

cmd/handler-utils.go

@@ -22,7 +22,6 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"os"
 	"strings"
 
 	"github.com/minio/minio/pkg/errors"
@@ -251,18 +250,18 @@ func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser,
 
 // Log headers and body.
 func httpTraceAll(f http.HandlerFunc) http.HandlerFunc {
-	if !globalHTTPTrace {
+	if globalHTTPTraceFile == nil {
 		return f
 	}
-	return httptracer.TraceReqHandlerFunc(f, os.Stdout, true)
+	return httptracer.TraceReqHandlerFunc(f, globalHTTPTraceFile, true)
 }
 
 // Log only the headers.
 func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
-	if !globalHTTPTrace {
+	if globalHTTPTraceFile == nil {
 		return f
 	}
-	return httptracer.TraceReqHandlerFunc(f, os.Stdout, false)
+	return httptracer.TraceReqHandlerFunc(f, globalHTTPTraceFile, false)
 }
 
 // Returns "/bucketName/objectName" for path-style or virtual-host-style requests.

cmd/jwt.go

@@ -63,14 +63,11 @@ func authenticateJWT(accessKey, secretKey string, expiry time.Duration) (string,
 		return "", errAuthentication
 	}
 
-	utcNow := UTCNow()
-	token := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{
-		ExpiresAt: utcNow.Add(expiry).Unix(),
-		IssuedAt:  utcNow.Unix(),
+	jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{
+		ExpiresAt: UTCNow().Add(expiry).Unix(),
 		Subject:   accessKey,
 	})
-
-	return token.SignedString([]byte(serverCred.SecretKey))
+	return jwt.SignedString([]byte(serverCred.SecretKey))
 }
 
 func authenticateNode(accessKey, secretKey string) (string, error) {
@@ -127,7 +124,7 @@ func webRequestAuthenticate(req *http.Request) error {
 		return errAuthentication
 	}
 	if err = claims.Valid(); err != nil {
-		return err
+		return errAuthentication
 	}
 	if claims.Subject != globalServerConfig.GetCredential().AccessKey {
 		return errInvalidAccessKeyID

cmd/lock-instrument.go

@@ -201,8 +201,6 @@ func (n *nsLockMap) statusBlockedToNone(param nsParam, lockSource, opsID string,
 	if lockInfo.status != blockedStatus {
 		return errors.Trace(LockInfoStateNotBlocked{param.volume, param.path, opsID})
 	}
-	// Clear the status by removing the entry for the given `opsID`.
-	delete(n.debugLockMap[param].lockInfo, opsID)
 
 	// Update global lock stats.
 	n.counters.lockTimedOut()

cmd/lock-instrument_test.go

@@ -637,3 +637,33 @@ func TestNsLockMapDeleteLockInfoEntryForVolumePath(t *testing.T) {
 		t.Errorf("Expected the count of all locks to be %v, but got %v", 0, globalNSMutex.counters.total)
 	}
 }
+
+// Test to assert that status change from blocked to none shouldn't remove lock info entry for ops
+// Ref: Logs from https://github.com/minio/minio/issues/5311
+func TestStatusBlockedToNone(t *testing.T) {
+	// Initialize namespace lock subsystem
+	initNSLock(false)
+
+	ns := globalNSMutex
+
+	volume, path := "bucket", "object"
+	param := nsParam{volume: volume, path: path}
+	lockSrc := "main.go:1"
+	opsID := "1"
+
+	err := ns.statusNoneToBlocked(param, lockSrc, opsID, false)
+	if err != nil {
+		t.Fatal("Failed to mark lock state to blocked")
+	}
+
+	err = ns.statusBlockedToNone(param, lockSrc, opsID, false)
+	if err != nil {
+		t.Fatal("Failed to mark lock state to none")
+	}
+
+	err = ns.deleteLockInfoEntryForOps(param, opsID)
+	if err != nil {
+		t.Fatalf("Expected deleting of lock entry for %s to pass but got %v", opsID, err)
+	}
+
+}

cmd/lockinfo-handlers.go

@@ -66,47 +66,6 @@ type OpsLockState struct {
 
 // listLocksInfo - Fetches locks held on bucket, matching prefix held for longer than duration.
 func listLocksInfo(bucket, prefix string, duration time.Duration) []VolumeLockInfo {
-	globalNSMutex.lockMapMutex.Lock()
-	defer globalNSMutex.lockMapMutex.Unlock()
-
-	// Fetch current time once instead of fetching system time for every lock.
-	timeNow := UTCNow()
-	volumeLocks := []VolumeLockInfo{}
-
-	for param, debugLock := range globalNSMutex.debugLockMap {
-		if param.volume != bucket {
-			continue
-		}
-		// N B empty prefix matches all param.path.
-		if !hasPrefix(param.path, prefix) {
-			continue
-		}
-
-		volLockInfo := VolumeLockInfo{
-			Bucket:                param.volume,
-			Object:                param.path,
-			LocksOnObject:         debugLock.counters.total,
-			TotalBlockedLocks:     debugLock.counters.blocked,
-			LocksAcquiredOnObject: debugLock.counters.granted,
-		}
-		// Filter locks that are held on bucket, prefix.
-		for opsID, lockInfo := range debugLock.lockInfo {
-			// filter locks that were held for longer than duration.
-			elapsed := timeNow.Sub(lockInfo.since)
-			if elapsed < duration {
-				continue
-			}
-			// Add locks that are held for longer than duration.
-			volLockInfo.LockDetailsOnObject = append(volLockInfo.LockDetailsOnObject,
-				OpsLockState{
-					OperationID: opsID,
-					LockSource:  lockInfo.lockSource,
-					LockType:    lockInfo.lType,
-					Status:      lockInfo.status,
-					Since:       lockInfo.since,
-				})
-			volumeLocks = append(volumeLocks, volLockInfo)
-		}
-	}
-	return volumeLocks
+	locksInfo, _ := newObjectLayerFn().ListLocks(bucket, prefix, duration)
+	return locksInfo
 }

cmd/lockinfo-handlers_test.go

@@ -28,11 +28,22 @@ func TestListLocksInfo(t *testing.T) {
 	// instrumentation information.
 	isDistXL := false
 	initNSLock(isDistXL)
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		t.Errorf("Failed to initialize object layer")
+	}
+	var nsMutex *nsLockMap
+	switch objAPI.(type) {
+	case *fsObjects:
+		nsMutex = objAPI.(*fsObjects).nsMutex
+	case *xlObjects:
+		nsMutex = objAPI.(*xlObjects).nsMutex
+	}
 
 	// Acquire a few locks to populate lock instrumentation.
 	// Take 10 read locks on bucket1/prefix1/obj1
 	for i := 0; i < 10; i++ {
-		readLk := globalNSMutex.NewNSLock("bucket1", "prefix1/obj1")
+		readLk := nsMutex.NewNSLock("bucket1", "prefix1/obj1")
 		if readLk.GetRLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
 			t.Errorf("Failed to get read lock on iteration %d", i)
 		}
@@ -40,7 +51,7 @@ func TestListLocksInfo(t *testing.T) {
 
 	// Take write locks on bucket1/prefix/obj{11..19}
 	for i := 0; i < 10; i++ {
-		wrLk := globalNSMutex.NewNSLock("bucket1", fmt.Sprintf("prefix1/obj%d", 10+i))
+		wrLk := nsMutex.NewNSLock("bucket1", fmt.Sprintf("prefix1/obj%d", 10+i))
 		if wrLk.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
 			t.Errorf("Failed to get write lock on iteration %d", i)
 		}

cmd/logger.go

@@ -17,81 +17,71 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"path"
+	"os"
+	"path/filepath"
 	"runtime"
 	"strings"
+	"time"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/minio/mc/pkg/console"
 	"github.com/minio/minio/pkg/errors"
 )
 
 var log = NewLogger()
+var trimStrings []string
 
-// LogTarget - interface for log target.
-type LogTarget interface {
-	Fire(entry *logrus.Entry) error
-	String() string
+// Level type
+type Level int8
+
+// Enumerated level types
+const (
+	Error Level = iota + 1
+	Fatal
+)
+
+func (level Level) String() string {
+	var lvlStr string
+	switch level {
+	case Error:
+		lvlStr = "ERROR"
+	case Fatal:
+		lvlStr = "FATAL"
+	}
+	return lvlStr
 }
 
-// BaseLogTarget - base log target.
-type BaseLogTarget struct {
-	Enable    bool `json:"enable"`
-	formatter logrus.Formatter
+type logEntry struct {
+	Level   string   `json:"level"`
+	Message string   `json:"message"`
+	Time    string   `json:"time"`
+	Cause   string   `json:"cause"`
+	Trace   []string `json:"trace"`
 }
 
-// Logger - higher level logger.
+// Logger - for console messages
 type Logger struct {
-	logger        *logrus.Logger
-	consoleTarget ConsoleLogger
-	targets       []LogTarget
-	quiet         bool
+	quiet bool
+	json  bool
 }
 
-// AddTarget - add logger to this hook.
-func (log *Logger) AddTarget(logTarget LogTarget) {
-	log.targets = append(log.targets, logTarget)
+// NewLogger - to create a new Logger object
+func NewLogger() *Logger {
+	return &Logger{}
 }
 
-// SetConsoleTarget - sets console target to this hook.
-func (log *Logger) SetConsoleTarget(consoleTarget ConsoleLogger) {
-	log.consoleTarget = consoleTarget
-}
-
-// Fire - log entry handler to save logs.
-func (log *Logger) Fire(entry *logrus.Entry) (err error) {
-	if err = log.consoleTarget.Fire(entry); err != nil {
-		log.Printf("Unable to log to console target. %s\n", err)
-	}
-
-	for _, logTarget := range log.targets {
-		if err = logTarget.Fire(entry); err != nil {
-			log.Printf("Unable to log to target %s. %s\n", logTarget, err)
-		}
-	}
-
-	return err
-}
-
-// Levels - returns list of log levels support.
-func (log *Logger) Levels() []logrus.Level {
-	return []logrus.Level{
-		logrus.PanicLevel,
-		logrus.FatalLevel,
-		logrus.ErrorLevel,
-		logrus.WarnLevel,
-		logrus.InfoLevel,
-		logrus.DebugLevel,
-	}
-}
-
-// EnableQuiet - sets quiet option.
+// EnableQuiet - turns quiet option on.
 func (log *Logger) EnableQuiet() {
 	log.quiet = true
 }
 
+// EnableJSON - outputs logs in json format.
+func (log *Logger) EnableJSON() {
+	log.json = true
+	log.quiet = true
+}
+
 // Println - wrapper to console.Println() with quiet flag.
 func (log *Logger) Println(args ...interface{}) {
 	if !log.quiet {
@@ -106,38 +96,66 @@ func (log *Logger) Printf(format string, args ...interface{}) {
 	}
 }
 
-// NewLogger - returns a new initialized logger.
-func NewLogger() *Logger {
-	logger := logrus.New()
-	logger.Out = ioutil.Discard
-	logger.Level = logrus.DebugLevel
-
-	l := &Logger{
-		logger:        logger,
-		consoleTarget: NewConsoleLogger(),
-	}
-
-	// Adds a console logger.
-	logger.Hooks.Add(l)
-
-	return l
-}
-
-func getSource() string {
-	var funcName string
-	pc, filename, lineNum, ok := runtime.Caller(2)
-	if ok {
-		filename = path.Base(filename)
-		funcName = strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "github.com/minio/minio/cmd.")
+func init() {
+	var goPathList []string
+	// Add all possible GOPATH paths into trimStrings
+	// Split GOPATH depending on the OS type
+	if runtime.GOOS == "windows" {
+		goPathList = strings.Split(GOPATH, ";")
 	} else {
-		filename = "<unknown>"
-		lineNum = 0
+		// All other types of OSs
+		goPathList = strings.Split(GOPATH, ":")
 	}
 
-	return fmt.Sprintf("[%s:%d:%s()]", filename, lineNum, funcName)
+	// Add trim string "{GOROOT}/src/" into trimStrings
+	trimStrings = []string{filepath.Join(runtime.GOROOT(), "src") + string(filepath.Separator)}
+
+	// Add all possible path from GOPATH=path1:path2...:pathN
+	// as "{path#}/src/" into trimStrings
+	for _, goPathString := range goPathList {
+		trimStrings = append(trimStrings, filepath.Join(goPathString, "src")+string(filepath.Separator))
+	}
+	// Add "github.com/minio/minio" as the last to cover
+	// paths like "{GOROOT}/src/github.com/minio/minio"
+	// and "{GOPATH}/src/github.com/minio/minio"
+	trimStrings = append(trimStrings, filepath.Join("github.com", "minio", "minio")+string(filepath.Separator))
 }
 
-func logIf(level logrus.Level, source string, err error, msg string, data ...interface{}) {
+func trimTrace(f string) string {
+	for _, trimString := range trimStrings {
+		f = strings.TrimPrefix(filepath.ToSlash(f), filepath.ToSlash(trimString))
+	}
+	return filepath.FromSlash(f)
+}
+
+// getTrace method - creates and returns stack trace
+func getTrace(traceLevel int) []string {
+	var trace []string
+	pc, file, lineNumber, ok := runtime.Caller(traceLevel)
+
+	for ok {
+		// Clean up the common prefixes
+		file = trimTrace(file)
+		// Get the function name
+		_, funcName := filepath.Split(runtime.FuncForPC(pc).Name())
+		// Skip duplicate traces that start with file name, "<autogenerated>"
+		// and also skip traces with function name that starts with "runtime."
+		if !strings.HasPrefix(file, "<autogenerated>") &&
+			!strings.HasPrefix(funcName, "runtime.") {
+			// Form and append a line of stack trace into a
+			// collection, 'trace', to build full stack trace
+			trace = append(trace, fmt.Sprintf("%v:%v:%v()", file, lineNumber, funcName))
+		}
+		traceLevel++
+		// Read stack trace information from PC
+		pc, file, lineNumber, ok = runtime.Caller(traceLevel)
+	}
+	return trace
+}
+
+func logIf(level Level, err error, msg string,
+	data ...interface{}) {
+
 	isErrIgnored := func(err error) (ok bool) {
 		err = errors.Cause(err)
 		switch err.(type) {
@@ -154,36 +172,51 @@ func logIf(level logrus.Level, source string, err error, msg string, data ...int
 	if err == nil || isErrIgnored(err) {
 		return
 	}
+	cause := strings.Title(err.Error())
+	// Get full stack trace
+	trace := getTrace(3)
+	// Get time
+	timeOfError := UTCNow().Format(time.RFC3339Nano)
+	// Output the formatted log message at console
+	var output string
+	message := fmt.Sprintf(msg, data...)
+	if log.json {
+		logJSON, err := json.Marshal(&logEntry{
+			Level:   level.String(),
+			Message: message,
+			Time:    timeOfError,
+			Cause:   cause,
+			Trace:   trace,
+		})
+		if err != nil {
+			panic("json marshal of logEntry failed: " + err.Error())
+		}
+		output = string(logJSON)
+	} else {
+		// Add a sequence number and formatting for each stack trace
+		// No formatting is required for the first entry
+		trace[0] = "1: " + trace[0]
+		for i, element := range trace[1:] {
+			trace[i+1] = fmt.Sprintf("%8v: %s", i+2, element)
+		}
+		errMsg := fmt.Sprintf("[%s] [%s] %s (%s)",
+			timeOfError, level.String(), message, cause)
 
-	fields := logrus.Fields{
-		"source": source,
-		"cause":  err.Error(),
+		output = fmt.Sprintf("\nTrace: %s\n%s",
+			strings.Join(trace, "\n"),
+			colorRed(colorBold(errMsg)))
 	}
+	fmt.Println(output)
 
-	if terr, ok := err.(*errors.Error); ok {
-		fields["stack"] = strings.Join(terr.Stack(), " ")
-	}
-
-	switch level {
-	case logrus.PanicLevel:
-		log.logger.WithFields(fields).Panicf(msg, data...)
-	case logrus.FatalLevel:
-		log.logger.WithFields(fields).Fatalf(msg, data...)
-	case logrus.ErrorLevel:
-		log.logger.WithFields(fields).Errorf(msg, data...)
-	case logrus.WarnLevel:
-		log.logger.WithFields(fields).Warnf(msg, data...)
-	case logrus.InfoLevel:
-		log.logger.WithFields(fields).Infof(msg, data...)
-	default:
-		log.logger.WithFields(fields).Debugf(msg, data...)
+	if level == Fatal {
+		os.Exit(1)
 	}
 }
 
 func errorIf(err error, msg string, data ...interface{}) {
-	logIf(logrus.ErrorLevel, getSource(), err, msg, data...)
+	logIf(Error, err, msg, data...)
 }
 
 func fatalIf(err error, msg string, data ...interface{}) {
-	logIf(logrus.FatalLevel, getSource(), err, msg, data...)
+	logIf(Fatal, err, msg, data...)
 }

cmd/logger_test.go

@@ -1,31 +0,0 @@
-/*
- * Minio Cloud Storage (C) 2015, 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"testing"
-)
-
-// Tests getSource().
-func TestGetSource(t *testing.T) {
-	currentSource := func() string { return getSource() }
-	gotSource := currentSource()
-	expectedSource := "[logger_test.go:26:TestGetSource()]"
-	if gotSource != expectedSource {
-		t.Errorf("expected : %s, got : %s", expectedSource, gotSource)
-	}
-}

cmd/main.go

@@ -38,6 +38,10 @@ var globalFlags = []cli.Flag{
 		Name:  "quiet",
 		Usage: "Disable startup information.",
 	},
+	cli.BoolFlag{
+		Name:  "json",
+		Usage: "Output server logs and startup information in json format.",
+	},
 }
 
 // Help template for minio.

cmd/namespace-lock.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"errors"
 	pathutil "path"
+	"runtime"
+	"strings"
 	"sync"
 
 	"fmt"
@@ -92,9 +94,9 @@ func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int)
 	return clnts, myNode
 }
 
-// initNSLock - initialize name space lock map.
-func initNSLock(isDistXL bool) {
-	globalNSMutex = &nsLockMap{
+// newNSLock - return a new name space lock map.
+func newNSLock(isDistXL bool) *nsLockMap {
+	nsMutex := nsLockMap{
 		isDistXL: isDistXL,
 		lockMap:  make(map[nsParam]*nsLock),
 		counters: &lockStat{},
@@ -102,7 +104,13 @@ func initNSLock(isDistXL bool) {
 
 	// Initialize nsLockMap with entry for instrumentation information.
 	// Entries of <volume,path> -> stateInfo of locks
-	globalNSMutex.debugLockMap = make(map[nsParam]*debugLockInfoPerVolumePath)
+	nsMutex.debugLockMap = make(map[nsParam]*debugLockInfoPerVolumePath)
+	return &nsMutex
+}
+
+// initNSLock - initialize name space lock map.
+func initNSLock(isDistXL bool) {
+	globalNSMutex = newNSLock(isDistXL)
 }
 
 // nsParam - carries name space resource.
@@ -360,3 +368,18 @@ func (li *lockInstance) RUnlock() {
 	readLock := true
 	li.ns.unlock(li.volume, li.path, li.opsID, readLock)
 }
+
+func getSource() string {
+	var funcName string
+	pc, filename, lineNum, ok := runtime.Caller(2)
+	if ok {
+		filename = pathutil.Base(filename)
+		funcName = strings.TrimPrefix(runtime.FuncForPC(pc).Name(),
+			"github.com/minio/minio/cmd.")
+	} else {
+		filename = "<unknown>"
+		lineNum = 0
+	}
+
+	return fmt.Sprintf("[%s:%d:%s()]", filename, lineNum, funcName)
+}

cmd/namespace-lock_test.go

@@ -21,6 +21,22 @@ import (
 	"time"
 )
 
+// WARNING:
+//
+// Expected source line number is hard coded, 32, in the
+// following test. Adding new code before this test or changing its
+// position will cause the line number to change and the test to FAIL
+// Tests getSource().
+func TestGetSource(t *testing.T) {
+	currentSource := func() string { return getSource() }
+	gotSource := currentSource()
+	// Hard coded line number, 32, in the "expectedSource" value
+	expectedSource := "[namespace-lock_test.go:32:TestGetSource()]"
+	if gotSource != expectedSource {
+		t.Errorf("expected : %s, got : %s", expectedSource, gotSource)
+	}
+}
+
 // Tests functionality provided by namespace lock.
 func TestNamespaceLockTest(t *testing.T) {
 	// List of test cases.

cmd/object-api-datatypes.go

@@ -45,8 +45,8 @@ type StorageInfo struct {
 		// Following fields are only meaningful if BackendType is Erasure.
 		OnlineDisks      int // Online disks during server startup.
 		OfflineDisks     int // Offline disks during server startup.
-		standardSCParity int // Parity disks for currently configured Standard storage class.
-		rrSCParity       int // Parity disks for currently configured Reduced Redundancy storage class.
+		StandardSCParity int // Parity disks for currently configured Standard storage class.
+		RRSCParity       int // Parity disks for currently configured Reduced Redundancy storage class.
 	}
 }
 

cmd/object-api-errors.go

@@ -349,6 +349,13 @@ func (e PartTooBig) Error() string {
 	return "Part size bigger than the allowed limit"
 }
 
+// InvalidETag error returned when the etag has changed on disk
+type InvalidETag struct{}
+
+func (e InvalidETag) Error() string {
+	return "etag of the object has changed"
+}
+
 // NotImplemented If a feature is not implemented
 type NotImplemented struct{}
 

cmd/object-api-getobject_test.go

@@ -152,7 +152,7 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	for i, testCase := range testCases {
-		err = obj.GetObject(testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer)
+		err = obj.GetObject(testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "")
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
 		}
@@ -262,7 +262,7 @@ func testGetObjectPermissionDenied(obj ObjectLayer, instanceType string, disks [
 			}
 		}
 
-		err = obj.GetObject(testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer)
+		err = obj.GetObject(testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "")
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
 		}
@@ -423,7 +423,7 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str
 	}
 
 	for i, testCase := range testCases {
-		err = obj.GetObject(testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer)
+		err = obj.GetObject(testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "")
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
 		}

cmd/object-api-interface.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"io"
+	"time"
 
 	"github.com/minio/minio/pkg/hash"
 )
@@ -36,16 +37,16 @@ type ObjectLayer interface {
 	ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 
 	// Object operations.
-	GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
+	GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error)
 	GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
 	PutObject(bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error)
-	CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error)
+	CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string, srcETag string) (objInfo ObjectInfo, err error)
 	DeleteObject(bucket, object string) error
 
 	// Multipart operations.
 	ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)
 	NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error)
-	CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, metadata map[string]string) (info PartInfo, err error)
+	CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, metadata map[string]string, srcEtag string) (info PartInfo, err error)
 	PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error)
 	ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error)
 	AbortMultipartUpload(bucket, object, uploadID string) error
@@ -56,6 +57,8 @@ type ObjectLayer interface {
 	ListBucketsHeal() (buckets []BucketInfo, err error)
 	HealObject(bucket, object string) (int, int, error)
 	ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error)
-	ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
-		delimiter string, maxUploads int) (ListMultipartsInfo, error)
+
+	// Locking operations
+	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
+	ClearLocks([]VolumeLockInfo) error
 }

cmd/object-api-multipart-common.go

@@ -169,7 +169,7 @@ func writeUploadJSON(u *uploadsV1, uploadsPath, tmpPath string, disk StorageAPI)
 }
 
 // listMultipartUploadIDs - list all the upload ids from a marker up to 'count'.
-func listMultipartUploadIDs(bucketName, objectName, uploadIDMarker string, count int, disk StorageAPI) ([]MultipartInfo, bool, error) {
+func (xl xlObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarker string, count int, disk StorageAPI) ([]MultipartInfo, bool, error) {
 	var uploads []MultipartInfo
 	// Read `uploads.json`.
 	uploadsJSON, err := readUploadsJSON(bucketName, objectName, disk)

cmd/object-api-utils.go

@@ -187,14 +187,26 @@ func getCompleteMultipartMD5(parts []CompletePart) (string, error) {
 	return s3MD5, nil
 }
 
-// Clean meta etag keys 'md5Sum', 'etag'.
-func cleanMetaETag(metadata map[string]string) map[string]string {
-	return cleanMetadata(metadata, "md5Sum", "etag")
+// Clean unwanted fields from metadata
+func cleanMetadata(metadata map[string]string) map[string]string {
+	// Remove STANDARD StorageClass
+	metadata = removeStandardStorageClass(metadata)
+	// Clean meta etag keys 'md5Sum', 'etag'.
+	return cleanMetadataKeys(metadata, "md5Sum", "etag")
 }
 
-// Clean metadata takes keys to be filtered
-// and returns a new map with the keys filtered.
-func cleanMetadata(metadata map[string]string, keyNames ...string) map[string]string {
+// Filter X-Amz-Storage-Class field only if it is set to STANDARD.
+// This is done since AWS S3 doesn't return STANDARD Storage class as response header.
+func removeStandardStorageClass(metadata map[string]string) map[string]string {
+	if metadata[amzStorageClass] == standardStorageClass {
+		delete(metadata, amzStorageClass)
+	}
+	return metadata
+}
+
+// cleanMetadataKeys takes keyNames to be filtered
+// and returns a new map with all the entries with keyNames removed.
+func cleanMetadataKeys(metadata map[string]string, keyNames ...string) map[string]string {
 	var newMeta = make(map[string]string)
 	for k, v := range metadata {
 		if contains(keyNames, k) {

cmd/object-api-utils_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"reflect"
 	"testing"
 )
 
@@ -197,3 +198,100 @@ func TestIsMinioMetaBucketName(t *testing.T) {
 		}
 	}
 }
+
+// Tests RemoveStandardStorageClass method. Expectation is metadata map
+// should be cleared of x-amz-storage-class, if it is set to STANDARD
+func TestRemoveStandardStorageClass(t *testing.T) {
+	tests := []struct {
+		name     string
+		metadata map[string]string
+		want     map[string]string
+	}{
+		{
+			name:     "1",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"},
+			want:     map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"},
+		},
+		{
+			name:     "2",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
+			want:     map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
+		},
+		{
+			name:     "3",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"},
+			want:     map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"},
+		},
+	}
+	for _, tt := range tests {
+		if got := removeStandardStorageClass(tt.metadata); !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
+		}
+	}
+}
+
+// Tests CleanMetadata method. Expectation is metadata map
+// should be cleared of etag, md5Sum and x-amz-storage-class, if it is set to STANDARD
+func TestCleanMetadata(t *testing.T) {
+	tests := []struct {
+		name     string
+		metadata map[string]string
+		want     map[string]string
+	}{
+		{
+			name:     "1",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"},
+			want:     map[string]string{"content-type": "application/octet-stream"},
+		},
+		{
+			name:     "2",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
+			want:     map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
+		},
+		{
+			name:     "3",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "md5Sum": "abcde"},
+			want:     map[string]string{"content-type": "application/octet-stream"},
+		},
+	}
+	for _, tt := range tests {
+		if got := cleanMetadata(tt.metadata); !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
+		}
+	}
+}
+
+// Tests CleanMetadataKeys method. Expectation is metadata map
+// should be cleared of keys passed to CleanMetadataKeys method
+func TestCleanMetadataKeys(t *testing.T) {
+	tests := []struct {
+		name     string
+		metadata map[string]string
+		keys     []string
+		want     map[string]string
+	}{
+		{
+			name:     "1",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD", "md5": "abcde"},
+			keys:     []string{"etag", "md5"},
+			want:     map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "STANDARD"},
+		},
+		{
+			name:     "2",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY", "md5sum": "abcde"},
+			keys:     []string{"etag", "md5sum"},
+			want:     map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
+		},
+		{
+			name:     "3",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "xyz": "abcde"},
+			keys:     []string{"etag", "xyz"},
+			want:     map[string]string{"content-type": "application/octet-stream"},
+		},
+	}
+	for _, tt := range tests {
+		if got := cleanMetadataKeys(tt.metadata, tt.keys...); !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
+		}
+	}
+}

cmd/object-handlers-common.go

@@ -230,12 +230,6 @@ func isETagEqual(left, right string) bool {
 // is a common function to be called from object handlers and
 // web handlers.
 func deleteObject(obj ObjectLayer, bucket, object string, r *http.Request) (err error) {
-	// Acquire a write lock before deleting the object.
-	objectLock := globalNSMutex.NewNSLock(bucket, object)
-	if err = objectLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objectLock.Unlock()
 
 	// Proceed to delete the object.
 	if err = obj.DeleteObject(bucket, object); err != nil {

cmd/object-handlers.go

@@ -107,17 +107,8 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// Lock the object before reading.
-	objectLock := globalNSMutex.NewNSLock(bucket, object)
-	if objectLock.GetRLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
-	defer objectLock.RUnlock()
-
 	objInfo, err := objectAPI.GetObjectInfo(bucket, object)
 	if err != nil {
-		errorIf(err, "Unable to fetch object info.")
 		apiErr := toAPIErrorCode(err)
 		if apiErr == ErrNoSuchKey {
 			apiErr = errAllowableObjectNotFound(bucket, r)
@@ -183,7 +174,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 
 	httpWriter := ioutil.WriteOnClose(writer)
 	// Reads the object at startOffset and writes to mw.
-	if err = objectAPI.GetObject(bucket, object, startOffset, length, httpWriter); err != nil {
+	if err = objectAPI.GetObject(bucket, object, startOffset, length, httpWriter, objInfo.ETag); err != nil {
 		errorIf(err, "Unable to write to client.")
 		if !httpWriter.HasWritten() { // write error response only if no data has been written to client yet
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -235,17 +226,8 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Lock the object before reading.
-	objectLock := globalNSMutex.NewNSLock(bucket, object)
-	if objectLock.GetRLock(globalObjectTimeout) != nil {
-		writeErrorResponseHeadersOnly(w, ErrOperationTimedOut)
-		return
-	}
-	defer objectLock.RUnlock()
-
 	objInfo, err := objectAPI.GetObjectInfo(bucket, object)
 	if err != nil {
-		errorIf(err, "Unable to fetch object info.")
 		apiErr := toAPIErrorCode(err)
 		if apiErr == ErrNoSuchKey {
 			apiErr = errAllowableObjectNotFound(bucket, r)
@@ -364,36 +346,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
 		return
 	}
-
 	cpSrcDstSame := srcBucket == dstBucket && srcObject == dstObject
-	// Hold write lock on destination since in both cases
-	// - if source and destination are same
-	// - if source and destination are different
-	// it is the sole mutating state.
-	objectDWLock := globalNSMutex.NewNSLock(dstBucket, dstObject)
-	if objectDWLock.GetLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
-	defer objectDWLock.Unlock()
-
-	// if source and destination are different, we have to hold
-	// additional read lock as well to protect against writes on
-	// source.
-	if !cpSrcDstSame {
-		// Hold read locks on source object only if we are
-		// going to read data from source object.
-		objectSRLock := globalNSMutex.NewNSLock(srcBucket, srcObject)
-		if objectSRLock.GetRLock(globalObjectTimeout) != nil {
-			writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-			return
-		}
-		defer objectSRLock.RUnlock()
-	}
 
 	objInfo, err := objectAPI.GetObjectInfo(srcBucket, srcObject)
 	if err != nil {
-		errorIf(err, "Unable to fetch object info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -427,7 +383,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// Copy source object to destination, if source and destination
 	// object is same then only metadata is updated.
-	objInfo, err = objectAPI.CopyObject(srcBucket, srcObject, dstBucket, dstObject, newMetadata)
+	objInfo, err = objectAPI.CopyObject(srcBucket, srcObject, dstBucket, dstObject, newMetadata, objInfo.ETag)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -488,7 +444,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	// Get Content-Md5 sent by client and verify if valid
 	md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5"))
 	if err != nil {
-		errorIf(err, "Unable to validate content-md5 format.")
 		writeErrorResponse(w, ErrInvalidDigest, r.URL)
 		return
 	}
@@ -500,7 +455,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		sizeStr := r.Header.Get("x-amz-decoded-content-length")
 		size, err = strconv.ParseInt(sizeStr, 10, 64)
 		if err != nil {
-			errorIf(err, "Unable to parse `x-amz-decoded-content-length` into its integer value", sizeStr)
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 			return
 		}
@@ -543,14 +497,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	// Lock the object.
-	objectLock := globalNSMutex.NewNSLock(bucket, object)
-	if objectLock.GetLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
-	defer objectLock.Unlock()
-
 	var (
 		md5hex    = hex.EncodeToString(md5Bytes)
 		sha256hex = ""
@@ -585,6 +531,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			writeErrorResponse(w, s3Err, r.URL)
 			return
 		}
+
 	case authTypePresigned, authTypeSigned:
 		if s3Err = reqSignatureV4Verify(r, globalServerConfig.GetRegion()); s3Err != ErrNone {
 			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
@@ -618,7 +565,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	objInfo, err := objectAPI.PutObject(bucket, object, hashReader, metadata)
 	if err != nil {
-		errorIf(err, "Unable to create an object. %s", r.URL.Path)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -691,7 +637,6 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 
 	uploadID, err := objectAPI.NewMultipartUpload(bucket, object, metadata)
 	if err != nil {
-		errorIf(err, "Unable to initiate new multipart upload id.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -755,18 +700,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Hold read locks on source object only if we are
-	// going to read data from source object.
-	objectSRLock := globalNSMutex.NewNSLock(srcBucket, srcObject)
-	if objectSRLock.GetRLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
-	defer objectSRLock.RUnlock()
-
 	objInfo, err := objectAPI.GetObjectInfo(srcBucket, srcObject)
 	if err != nil {
-		errorIf(err, "Unable to fetch object info.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -806,9 +741,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	// Copy source object to destination, if source and destination
 	// object is same then only metadata is updated.
 	partInfo, err := objectAPI.CopyObjectPart(srcBucket, srcObject, dstBucket,
-		dstObject, uploadID, partID, startOffset, length, nil)
+		dstObject, uploadID, partID, startOffset, length, nil, objInfo.ETag)
 	if err != nil {
-		errorIf(err, "Unable to perform CopyObjectPart %s/%s", srcBucket, srcObject)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -860,7 +794,6 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		sizeStr := r.Header.Get("x-amz-decoded-content-length")
 		size, err = strconv.ParseInt(sizeStr, 10, 64)
 		if err != nil {
-			errorIf(err, "Unable to parse `x-amz-decoded-content-length` into its integer value", sizeStr)
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 			return
 		}
@@ -939,7 +872,6 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 	hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex)
 	if err != nil {
-		errorIf(err, "Unable to create object part.")
 		// Verify if the underlying error is signature mismatch.
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -947,7 +879,6 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 	partInfo, err := objectAPI.PutObjectPart(bucket, object, uploadID, partID, hashReader)
 	if err != nil {
-		errorIf(err, "Unable to create object part.")
 		// Verify if the underlying error is signature mismatch.
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -978,7 +909,6 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 
 	uploadID, _, _, _ := getObjectResources(r.URL.Query())
 	if err := objectAPI.AbortMultipartUpload(bucket, object, uploadID); err != nil {
-		errorIf(err, "Unable to abort multipart upload.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -1013,7 +943,6 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 	}
 	listPartsInfo, err := objectAPI.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
 	if err != nil {
-		errorIf(err, "Unable to list uploaded parts.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -1046,13 +975,11 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	completeMultipartBytes, err := goioutil.ReadAll(r.Body)
 	if err != nil {
-		errorIf(err, "Unable to complete multipart upload.")
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
 	complMultipartUpload := &CompleteMultipartUpload{}
 	if err = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); err != nil {
-		errorIf(err, "Unable to parse complete multipart upload XML.")
 		writeErrorResponse(w, ErrMalformedXML, r.URL)
 		return
 	}
@@ -1072,17 +999,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		completeParts = append(completeParts, part)
 	}
 
-	// Hold write lock on the object.
-	destLock := globalNSMutex.NewNSLock(bucket, object)
-	if destLock.GetLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
-	defer destLock.Unlock()
-
 	objInfo, err := objectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts)
 	if err != nil {
-		errorIf(err, "Unable to complete multipart upload.")
 		err = errors.Cause(err)
 		switch oErr := err.(type) {
 		case PartTooSmall:
@@ -1101,7 +1019,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	response := generateCompleteMultpartUploadResponse(bucket, object, location, objInfo.ETag)
 	encodedSuccessResponse := encodeResponse(response)
 	if err != nil {
-		errorIf(err, "Unable to parse CompleteMultipartUpload response")
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}

cmd/object-handlers_test.go

@@ -766,7 +766,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 				t.Fatalf("Test %d: %s: ContentEncoding is set to \"%s\" which is unexpected, expected \"%s\"", i+1, instanceType, objInfo.ContentEncoding, expectedContentEncoding)
 			}
 			buffer := new(bytes.Buffer)
-			err = obj.GetObject(testCase.bucketName, testCase.objectName, 0, int64(testCase.dataLen), buffer)
+			err = obj.GetObject(testCase.bucketName, testCase.objectName, 0, int64(testCase.dataLen), buffer, objInfo.ETag)
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
 			}
@@ -943,7 +943,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			buffer := new(bytes.Buffer)
 
 			// Fetch the object to check whether the content is same as the one uploaded via PutObject.
-			err = obj.GetObject(testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer)
+			err = obj.GetObject(testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer, "")
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
 			}
@@ -986,7 +986,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 		if testCase.expectedRespStatus == http.StatusOK {
 			buffer := new(bytes.Buffer)
 			// Fetch the object to check whether the content is same as the one uploaded via PutObject.
-			err = obj.GetObject(testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer)
+			err = obj.GetObject(testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer, "")
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
 			}
@@ -1138,7 +1138,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 	}
 
 	var buf bytes.Buffer
-	if err = obj.GetObject(bucketName, testObject, 0, int64(len(bytesData[0].byteData)), &buf); err != nil {
+	if err = obj.GetObject(bucketName, testObject, 0, int64(len(bytesData[0].byteData)), &buf, ""); err != nil {
 		t.Fatalf("Test: %s reading completed file failed: <ERROR> %v", instanceType, err)
 	}
 	if !bytes.Equal(buf.Bytes(), bytesData[0].byteData) {
@@ -1795,7 +1795,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		if rec.Code == http.StatusOK {
 			// See if the new object is formed.
 			// testing whether the copy was successful.
-			err = obj.GetObject(testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0])
+			err = obj.GetObject(testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "")
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
 			}

cmd/object_api_suite_test.go

@@ -207,7 +207,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
 
 	for key, value := range objects {
 		var byteBuffer bytes.Buffer
-		err = obj.GetObject("bucket", key, 0, int64(len(value)), &byteBuffer)
+		err = obj.GetObject("bucket", key, 0, int64(len(value)), &byteBuffer, "")
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -456,7 +456,7 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan
 	}
 
 	var bytesBuffer bytes.Buffer
-	err = obj.GetObject("bucket", "object", 0, length, &bytesBuffer)
+	err = obj.GetObject("bucket", "object", 0, length, &bytesBuffer, "")
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -523,7 +523,7 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	err = obj.GetObject("bucket", "object", 0, length, &bytesBuffer1)
+	err = obj.GetObject("bucket", "object", 0, length, &bytesBuffer1, "")
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -536,7 +536,7 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	err = obj.GetObject("bucket", "object", 0, length, &bytesBuffer2)
+	err = obj.GetObject("bucket", "object", 0, length, &bytesBuffer2, "")
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -566,7 +566,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle
 	}
 
 	var bytesBuffer bytes.Buffer
-	err = obj.GetObject("bucket", "dir1/dir2/object", 0, length, &bytesBuffer)
+	err = obj.GetObject("bucket", "dir1/dir2/object", 0, length, &bytesBuffer, "")
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}

cmd/os-reliable.go

@@ -0,0 +1,121 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path"
+)
+
+// Wrapper functions to os.MkdirAll, which calls reliableMkdirAll
+// this is to ensure that if there is a racy parent directory
+// delete in between we can simply retry the operation.
+func mkdirAll(dirPath string, mode os.FileMode) (err error) {
+	if dirPath == "" {
+		return errInvalidArgument
+	}
+
+	if err = checkPathLength(dirPath); err != nil {
+		return err
+	}
+
+	if err = reliableMkdirAll(dirPath, mode); err != nil {
+		// File path cannot be verified since one of the parents is a file.
+		if isSysErrNotDir(err) {
+			return errFileAccessDenied
+		} else if isSysErrPathNotFound(err) {
+			// This is a special case should be handled only for
+			// windows, because windows API does not return "not a
+			// directory" error message. Handle this specifically here.
+			return errFileAccessDenied
+		}
+	}
+	return err
+}
+
+// Reliably retries os.MkdirAll if for some reason os.MkdirAll returns
+// syscall.ENOENT (parent does not exist).
+func reliableMkdirAll(dirPath string, mode os.FileMode) (err error) {
+	i := 0
+	for {
+		// Creates all the parent directories, with mode 0777 mkdir honors system umask.
+		if err = os.MkdirAll(dirPath, mode); err != nil {
+			// Retry only for the first retryable error.
+			if os.IsNotExist(err) && i == 0 {
+				i++
+				continue
+			}
+		}
+		break
+	}
+	return err
+}
+
+// Wrapper function to os.Rename, which calls reliableMkdirAll
+// and reliableRenameAll. This is to ensure that if there is a
+// racy parent directory delete in between we can simply retry
+// the operation.
+func renameAll(srcFilePath, dstFilePath string) (err error) {
+	if srcFilePath == "" || dstFilePath == "" {
+		return errInvalidArgument
+	}
+
+	if err = checkPathLength(srcFilePath); err != nil {
+		return err
+	}
+	if err = checkPathLength(dstFilePath); err != nil {
+		return err
+	}
+
+	if err = reliableRename(srcFilePath, dstFilePath); err != nil {
+		if isSysErrNotDir(err) {
+			return errFileAccessDenied
+		} else if isSysErrPathNotFound(err) {
+			// This is a special case should be handled only for
+			// windows, because windows API does not return "not a
+			// directory" error message. Handle this specifically here.
+			return errFileAccessDenied
+		} else if isSysErrCrossDevice(err) {
+			return fmt.Errorf("%s (%s)->(%s)", errCrossDeviceLink, srcFilePath, dstFilePath)
+		} else if os.IsNotExist(err) {
+			return errFileNotFound
+		}
+	}
+	return err
+}
+
+// Reliably retries os.RenameAll if for some reason os.RenameAll returns
+// syscall.ENOENT (parent does not exist).
+func reliableRename(srcFilePath, dstFilePath string) (err error) {
+	i := 0
+	for {
+		if err = reliableMkdirAll(path.Dir(dstFilePath), 0777); err != nil {
+			return err
+		}
+		// After a successful parent directory create attempt a renameAll.
+		if err = os.Rename(srcFilePath, dstFilePath); err != nil {
+			// Retry only for the first retryable error.
+			if os.IsNotExist(err) && i == 0 {
+				i++
+				continue
+			}
+		}
+		break
+	}
+	return err
+}

cmd/os-reliable_test.go

@@ -0,0 +1,78 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"testing"
+
+	"github.com/minio/minio/pkg/errors"
+)
+
+// Tests - mkdirAll()
+func TestOSMkdirAll(t *testing.T) {
+	// create posix test setup
+	_, path, err := newPosixTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer os.RemoveAll(path)
+
+	if err = mkdirAll("", 0777); errors.Cause(err) != errInvalidArgument {
+		t.Fatal("Unexpected error", err)
+	}
+
+	if err = mkdirAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0777); errors.Cause(err) != errFileNameTooLong {
+		t.Fatal("Unexpected error", err)
+	}
+
+	if err = mkdirAll(pathJoin(path, "success-vol", "success-object"), 0777); err != nil {
+		t.Fatal("Unexpected error", err)
+	}
+}
+
+// Tests - renameAll()
+func TestOSRenameAll(t *testing.T) {
+	// create posix test setup
+	_, path, err := newPosixTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer os.RemoveAll(path)
+
+	if err = mkdirAll(pathJoin(path, "testvolume1"), 0777); err != nil {
+		t.Fatal(err)
+	}
+	if err = renameAll("", "foo"); err != errInvalidArgument {
+		t.Fatal(err)
+	}
+	if err = renameAll("foo", ""); err != errInvalidArgument {
+		t.Fatal(err)
+	}
+	if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil {
+		t.Fatal(err)
+	}
+	if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errors.Cause(err) != errFileNotFound {
+		t.Fatal(err)
+	}
+	if err = renameAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errors.Cause(err) != errFileNameTooLong {
+		t.Fatal("Unexpected error", err)
+	}
+	if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong {
+		t.Fatal("Unexpected error", err)
+	}
+}

cmd/posix.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -674,14 +674,7 @@ func (s *posix) createFile(volume, path string) (f *os.File, err error) {
 	} else {
 		// Create top level directories if they don't exist.
 		// with mode 0777 mkdir honors system umask.
-		if err = os.MkdirAll(slashpath.Dir(filePath), 0777); err != nil {
-			// File path cannot be verified since one of the parents is a file.
-			if isSysErrNotDir(err) {
-				return nil, errFileAccessDenied
-			} else if isSysErrPathNotFound(err) {
-				// Add specific case for windows.
-				return nil, errFileAccessDenied
-			}
+		if err = mkdirAll(slashpath.Dir(filePath), 0777); err != nil {
 			return nil, err
 		}
 	}
@@ -970,25 +963,8 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 		}
 		// Destination does not exist, hence proceed with the rename.
 	}
-	// Creates all the parent directories, with mode 0777 mkdir honors system umask.
-	if err = os.MkdirAll(slashpath.Dir(dstFilePath), 0777); err != nil {
-		// File path cannot be verified since one of the parents is a file.
-		if isSysErrNotDir(err) {
-			return errFileAccessDenied
-		} else if isSysErrPathNotFound(err) {
-			// This is a special case should be handled only for
-			// windows, because windows API does not return "not a
-			// directory" error message. Handle this specifically here.
-			return errFileAccessDenied
-		}
-		return err
-	}
-	// Finally attempt a rename.
-	err = os.Rename((srcFilePath), (dstFilePath))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return errFileNotFound
-		}
+
+	if err = renameAll(srcFilePath, dstFilePath); err != nil {
 		return err
 	}
 

cmd/postpolicyform_test.go

@@ -20,59 +20,73 @@ import (
 	"encoding/base64"
 	"net/http"
 	"testing"
+
+	minio "github.com/minio/minio-go"
 )
 
 // Test Post Policy parsing and checking conditions
 func TestPostPolicyForm(t *testing.T) {
+	pp := minio.NewPostPolicy()
+	pp.SetBucket("testbucket")
+	pp.SetContentType("image/jpeg")
+	pp.SetUserMetadata("uuid", "14365123651274")
+	pp.SetKeyStartsWith("user/user1/filename")
+	pp.SetContentLengthRange(1048579, 10485760)
+	pp.SetSuccessStatusAction("201")
+
 	type testCase struct {
-		Bucket                   string
-		Key                      string
-		ACL                      string
-		XAmzDate                 string
-		XAmzServerSideEncryption string
-		XAmzAlgorithm            string
-		XAmzCredential           string
-		XAmzMetaUUID             string
-		ContentType              string
-		SuccessActionRedirect    string
-		Policy                   string
-		ErrCode                  APIErrorCode
+		Bucket              string
+		Key                 string
+		XAmzDate            string
+		XAmzAlgorithm       string
+		XAmzCredential      string
+		XAmzMetaUUID        string
+		ContentType         string
+		SuccessActionStatus string
+		Policy              string
+		Expired             bool
+		ErrCode             APIErrorCode
 	}
+
 	testCases := []testCase{
 		// Everything is fine with this test
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", ACL: "public-read", XAmzServerSideEncryption: "AES256", XAmzMetaUUID: "14365123651274", SuccessActionRedirect: "http://127.0.0.1:9000/", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Policy: "eyAiZXhwaXJhdGlvbiI6ICIyMDE3LTEyLTMwVDEyOjAwOjAwLjAwMFoiLCAiY29uZGl0aW9ucyI6IFsgeyJidWNrZXQiOiAidGVzdGJ1Y2tldCJ9LCBbInN0YXJ0cy13aXRoIiwgIiRrZXkiLCAidXNlci91c2VyMS9maWxlbmFtZSJdLCB7ImFjbCI6ICJwdWJsaWMtcmVhZCJ9LCB7InN1Y2Nlc3NfYWN0aW9uX3JlZGlyZWN0IjogImh0dHA6Ly8xMjcuMC4wLjE6OTAwMC8ifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgImltYWdlLyJdLCBbImNvbnRlbnQtbGVuZ3RoLXJhbmdlIiwgMTA0ODU3OSwgMTA0ODU3NjBdLCB7IngtYW16LW1ldGEtdXVpZCI6ICIxNDM2NTEyMzY1MTI3NCJ9LCB7IngtYW16LXNlcnZlci1zaWRlLWVuY3J5cHRpb24iOiAiQUVTMjU2In0sIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LW1ldGEtdGFnIiwgIiJdLCB7IngtYW16LWNyZWRlbnRpYWwiOiAiS1ZHS01EVVEyM1RDWlhUTFRITFAvMjAxNjA3MjcvdXMtZWFzdC0xL3MzL2F3czRfcmVxdWVzdCJ9LCB7IngtYW16LWFsZ29yaXRobSI6ICJBV1M0LUhNQUMtU0hBMjU2In0sIHsieC1hbXotZGF0ZSI6ICIyMDE2MDcyN1QwMDAwMDBaIiB9IF0gfQ==", ErrCode: ErrNone},
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrNone},
 		// Expired policy document
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", ACL: "public-read", XAmzServerSideEncryption: "AES256", XAmzMetaUUID: "14365123651274", SuccessActionRedirect: "http://127.0.0.1:9000/", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Policy: "eyAiZXhwaXJhdGlvbiI6ICIyMDE1LTEyLTMwVDEyOjAwOjAwLjAwMFoiLCAiY29uZGl0aW9ucyI6IFsgeyJidWNrZXQiOiAidGVzdGJ1Y2tldCJ9LCBbInN0YXJ0cy13aXRoIiwgIiRrZXkiLCAidXNlci91c2VyMS9maWxlbmFtZSJdLCB7ImFjbCI6ICJwdWJsaWMtcmVhZCJ9LCB7InN1Y2Nlc3NfYWN0aW9uX3JlZGlyZWN0IjogImh0dHA6Ly8xMjcuMC4wLjE6OTAwMC8ifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgImltYWdlLyJdLCBbImNvbnRlbnQtbGVuZ3RoLXJhbmdlIiwgMTA0ODU3OSwgMTA0ODU3NjBdLCB7IngtYW16LW1ldGEtdXVpZCI6ICIxNDM2NTEyMzY1MTI3NCJ9LCB7IngtYW16LXNlcnZlci1zaWRlLWVuY3J5cHRpb24iOiAiQUVTMjU2In0sIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LW1ldGEtdGFnIiwgIiJdLCB7IngtYW16LWNyZWRlbnRpYWwiOiAiS1ZHS01EVVEyM1RDWlhUTFRITFAvMjAxNjA3MjcvdXMtZWFzdC0xL3MzL2F3czRfcmVxdWVzdCJ9LCB7IngtYW16LWFsZ29yaXRobSI6ICJBV1M0LUhNQUMtU0hBMjU2In0sIHsieC1hbXotZGF0ZSI6ICIyMDE2MDcyN1QwMDAwMDBaIiB9IF0gfQo=", ErrCode: ErrPolicyAlreadyExpired},
-		// Passing AMZ date with starts-with operator which is fobidden
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", ACL: "public-read", XAmzServerSideEncryption: "AES256", XAmzMetaUUID: "14365123651274", SuccessActionRedirect: "http://127.0.0.1:9000/", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Policy: "eyAiZXhwaXJhdGlvbiI6ICIyMTE1LTEyLTMwVDEyOjAwOjAwLjAwMFoiLCAiY29uZGl0aW9ucyI6IFsgeyJidWNrZXQiOiAidGVzdGJ1Y2tldCJ9LCBbInN0YXJ0cy13aXRoIiwgIiRrZXkiLCAidXNlci91c2VyMS9maWxlbmFtZSJdLCB7ImFjbCI6ICJwdWJsaWMtcmVhZCJ9LCB7InN1Y2Nlc3NfYWN0aW9uX3JlZGlyZWN0IjogImh0dHA6Ly8xMjcuMC4wLjE6OTAwMC8ifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgImltYWdlLyJdLCBbImNvbnRlbnQtbGVuZ3RoLXJhbmdlIiwgMTA0ODU3OSwgMTA0ODU3NjBdLCB7IngtYW16LW1ldGEtdXVpZCI6ICIxNDM2NTEyMzY1MTI3NCJ9LCB7IngtYW16LXNlcnZlci1zaWRlLWVuY3J5cHRpb24iOiAiQUVTMjU2In0sIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LW1ldGEtdGFnIiwgIiJdLCB7IngtYW16LWNyZWRlbnRpYWwiOiAiS1ZHS01EVVEyM1RDWlhUTFRITFAvMjAxNjA3MjcvdXMtZWFzdC0xL3MzL2F3czRfcmVxdWVzdCJ9LCB7IngtYW16LWFsZ29yaXRobSI6ICJBV1M0LUhNQUMtU0hBMjU2In0sIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LWRhdGUiLCAiMjAxNjA3MjdUMDAwMDAwWiJdIF0gfQo=", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Expired: true, ErrCode: ErrPolicyAlreadyExpired},
 		// Different AMZ date
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", ACL: "public-read", XAmzServerSideEncryption: "AES256", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Policy: "eyAiZXhwaXJhdGlvbiI6ICIyMDE3LTEyLTMwVDEyOjAwOjAwLjAwMFoiLCAiY29uZGl0aW9ucyI6IFsgeyJidWNrZXQiOiAidGVzdGJ1Y2tldCJ9LCBbInN0YXJ0cy13aXRoIiwgIiRrZXkiLCAidXNlci91c2VyMS9maWxlbmFtZSJdLCB7ImFjbCI6ICJwdWJsaWMtcmVhZCJ9LCB7InN1Y2Nlc3NfYWN0aW9uX3JlZGlyZWN0IjogImh0dHA6Ly8xMjcuMC4wLjE6OTAwMC8ifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgImltYWdlLyJdLCB7IngtYW16LW1ldGEtdXVpZCI6ICIxNDM2NTEyMzY1MTI3NCJ9LCB7IngtYW16LXNlcnZlci1zaWRlLWVuY3J5cHRpb24iOiAiQUVTMjU2In0sIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LW1ldGEtdGFnIiwgIiJdLCB7IngtYW16LWNyZWRlbnRpYWwiOiAiS1ZHS01EVVEyM1RDWlhUTFRITFAvMjAxNjA3MjcvdXMtZWFzdC0xL3MzL2F3czRfcmVxdWVzdCJ9LCB7IngtYW16LWFsZ29yaXRobSI6ICJBV1M0LUhNQUMtU0hBMjU2In0sIHsieC1hbXotZGF0ZSI6ICIyMDIwMDcyN1QwMDAwMDBaIiB9IF0gfQ==", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrAccessDenied},
 		// Key which doesn't start with user/user1/filename
-		{Bucket: "testbucket", Key: "myfile.txt", XAmzDate: "20160727T000000Z", ACL: "public-read", XAmzServerSideEncryption: "AES256", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Policy: "eyAiZXhwaXJhdGlvbiI6ICIyMDE3LTEyLTMwVDEyOjAwOjAwLjAwMFoiLCAiY29uZGl0aW9ucyI6IFsgeyJidWNrZXQiOiAidGVzdGJ1Y2tldCJ9LCBbInN0YXJ0cy13aXRoIiwgIiRrZXkiLCAidXNlci91c2VyMS9maWxlbmFtZSJdLCB7ImFjbCI6ICJwdWJsaWMtcmVhZCJ9LCB7InN1Y2Nlc3NfYWN0aW9uX3JlZGlyZWN0IjogImh0dHA6Ly8xMjcuMC4wLjE6OTAwMC8ifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgImltYWdlLyJdLCB7IngtYW16LW1ldGEtdXVpZCI6ICIxNDM2NTEyMzY1MTI3NCJ9LCB7IngtYW16LXNlcnZlci1zaWRlLWVuY3J5cHRpb24iOiAiQUVTMjU2In0sIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LW1ldGEtdGFnIiwgIiJdLCB7IngtYW16LWNyZWRlbnRpYWwiOiAiS1ZHS01EVVEyM1RDWlhUTFRITFAvMjAxNjA3MjcvdXMtZWFzdC0xL3MzL2F3czRfcmVxdWVzdCJ9LCB7IngtYW16LWFsZ29yaXRobSI6ICJBV1M0LUhNQUMtU0hBMjU2In0sIHsieC1hbXotZGF0ZSI6ICIyMDIwMDcyN1QwMDAwMDBaIiB9IF0gfQ==", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "myfile.txt", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrAccessDenied},
 		// Incorrect bucket name.
-		{Bucket: "incorrect", Key: "user/user1/filename/myfile.txt", ACL: "public-read", XAmzServerSideEncryption: "AES256", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Policy: "eyAiZXhwaXJhdGlvbiI6ICIyMDE3LTEyLTMwVDEyOjAwOjAwLjAwMFoiLCAiY29uZGl0aW9ucyI6IFsgeyJidWNrZXQiOiAidGVzdGJ1Y2tldCJ9LCBbInN0YXJ0cy13aXRoIiwgIiRrZXkiLCAidXNlci91c2VyMS9maWxlbmFtZSJdLCB7ImFjbCI6ICJwdWJsaWMtcmVhZCJ9LCB7InN1Y2Nlc3NfYWN0aW9uX3JlZGlyZWN0IjogImh0dHA6Ly8xMjcuMC4wLjE6OTAwMC8ifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgImltYWdlLyJdLCBbImNvbnRlbnQtbGVuZ3RoLXJhbmdlIiwgMTA0ODU3OSwgMTA0ODU3NjBdLCB7IngtYW16LW1ldGEtdXVpZCI6ICIxNDM2NTEyMzY1MTI3NCJ9LCB7IngtYW16LXNlcnZlci1zaWRlLWVuY3J5cHRpb24iOiAiQUVTMjU2In0sIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LW1ldGEtdGFnIiwgIiJdLCB7IngtYW16LWNyZWRlbnRpYWwiOiAiS1ZHS01EVVEyM1RDWlhUTFRITFAvMjAxNjA3MjcvdXMtZWFzdC0xL3MzL2F3czRfcmVxdWVzdCJ9LCB7IngtYW16LWFsZ29yaXRobSI6ICJBV1M0LUhNQUMtU0hBMjU2In0sIHsieC1hbXotZGF0ZSI6ICIyMDE2MDcyN1QwMDAwMDBaIiB9IF0gfQ==", ErrCode: ErrAccessDenied},
+		{Bucket: "incorrect", Key: "user/user1/filename/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrAccessDenied},
 		// Incorrect key name
-		{Bucket: "testbucket", Key: "incorrect", ACL: "public-read", XAmzDate: "20160727T000000Z", XAmzServerSideEncryption: "AES256", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Policy: "eyAiZXhwaXJhdGlvbiI6ICIyMDE3LTEyLTMwVDEyOjAwOjAwLjAwMFoiLCAiY29uZGl0aW9ucyI6IFsgeyJidWNrZXQiOiAidGVzdGJ1Y2tldCJ9LCBbInN0YXJ0cy13aXRoIiwgIiRrZXkiLCAidXNlci91c2VyMS9maWxlbmFtZSJdLCB7ImFjbCI6ICJwdWJsaWMtcmVhZCJ9LCB7InN1Y2Nlc3NfYWN0aW9uX3JlZGlyZWN0IjogImh0dHA6Ly8xMjcuMC4wLjE6OTAwMC8ifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgImltYWdlLyJdLCBbImNvbnRlbnQtbGVuZ3RoLXJhbmdlIiwgMTA0ODU3OSwgMTA0ODU3NjBdLCB7IngtYW16LW1ldGEtdXVpZCI6ICIxNDM2NTEyMzY1MTI3NCJ9LCB7IngtYW16LXNlcnZlci1zaWRlLWVuY3J5cHRpb24iOiAiQUVTMjU2In0sIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LW1ldGEtdGFnIiwgIiJdLCB7IngtYW16LWNyZWRlbnRpYWwiOiAiS1ZHS01EVVEyM1RDWlhUTFRITFAvMjAxNjA3MjcvdXMtZWFzdC0xL3MzL2F3czRfcmVxdWVzdCJ9LCB7IngtYW16LWFsZ29yaXRobSI6ICJBV1M0LUhNQUMtU0hBMjU2In0sIHsieC1hbXotZGF0ZSI6ICIyMDE2MDcyN1QwMDAwMDBaIiB9IF0gfQ==", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "incorrect", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrAccessDenied},
 		// Incorrect date
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", ACL: "public-read", XAmzServerSideEncryption: "AES256", XAmzMetaUUID: "14365123651274", XAmzDate: "incorrect", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Policy: "eyAiZXhwaXJhdGlvbiI6ICIyMDE3LTEyLTMwVDEyOjAwOjAwLjAwMFoiLCAiY29uZGl0aW9ucyI6IFsgeyJidWNrZXQiOiAidGVzdGJ1Y2tldCJ9LCBbInN0YXJ0cy13aXRoIiwgIiRrZXkiLCAidXNlci91c2VyMS9maWxlbmFtZSJdLCB7ImFjbCI6ICJwdWJsaWMtcmVhZCJ9LCB7InN1Y2Nlc3NfYWN0aW9uX3JlZGlyZWN0IjogImh0dHA6Ly8xMjcuMC4wLjE6OTAwMC8ifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgImltYWdlLyJdLCBbImNvbnRlbnQtbGVuZ3RoLXJhbmdlIiwgMTA0ODU3OSwgMTA0ODU3NjBdLCB7IngtYW16LW1ldGEtdXVpZCI6ICIxNDM2NTEyMzY1MTI3NCJ9LCB7IngtYW16LXNlcnZlci1zaWRlLWVuY3J5cHRpb24iOiAiQUVTMjU2In0sIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LW1ldGEtdGFnIiwgIiJdLCB7IngtYW16LWNyZWRlbnRpYWwiOiAiS1ZHS01EVVEyM1RDWlhUTFRITFAvMjAxNjA3MjcvdXMtZWFzdC0xL3MzL2F3czRfcmVxdWVzdCJ9LCB7IngtYW16LWFsZ29yaXRobSI6ICJBV1M0LUhNQUMtU0hBMjU2In0sIHsieC1hbXotZGF0ZSI6ICIyMDE2MDcyN1QwMDAwMDBaIiB9IF0gfQ==", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "incorrect", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrAccessDenied},
 		// Incorrect ContentType
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", ACL: "public-read", XAmzServerSideEncryption: "AES256", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "incorrect", Policy: "eyAiZXhwaXJhdGlvbiI6ICIyMDE3LTEyLTMwVDEyOjAwOjAwLjAwMFoiLCAiY29uZGl0aW9ucyI6IFsgeyJidWNrZXQiOiAidGVzdGJ1Y2tldCJ9LCBbInN0YXJ0cy13aXRoIiwgIiRrZXkiLCAidXNlci91c2VyMS9maWxlbmFtZSJdLCB7ImFjbCI6ICJwdWJsaWMtcmVhZCJ9LCB7InN1Y2Nlc3NfYWN0aW9uX3JlZGlyZWN0IjogImh0dHA6Ly8xMjcuMC4wLjE6OTAwMC8ifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgImltYWdlLyJdLCBbImNvbnRlbnQtbGVuZ3RoLXJhbmdlIiwgMTA0ODU3OSwgMTA0ODU3NjBdLCB7IngtYW16LW1ldGEtdXVpZCI6ICIxNDM2NTEyMzY1MTI3NCJ9LCB7IngtYW16LXNlcnZlci1zaWRlLWVuY3J5cHRpb24iOiAiQUVTMjU2In0sIFsic3RhcnRzLXdpdGgiLCAiJHgtYW16LW1ldGEtdGFnIiwgIiJdLCB7IngtYW16LWNyZWRlbnRpYWwiOiAiS1ZHS01EVVEyM1RDWlhUTFRITFAvMjAxNjA3MjcvdXMtZWFzdC0xL3MzL2F3czRfcmVxdWVzdCJ9LCB7IngtYW16LWFsZ29yaXRobSI6ICJBV1M0LUhNQUMtU0hBMjU2In0sIHsieC1hbXotZGF0ZSI6ICIyMDE2MDcyN1QwMDAwMDBaIiB9IF0gfQ==", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "incorrect", ErrCode: ErrAccessDenied},
 	}
 	// Validate all the test cases.
 	for i, tt := range testCases {
 		formValues := make(http.Header)
 		formValues.Set("Bucket", tt.Bucket)
-		formValues.Set("Acl", tt.ACL)
 		formValues.Set("Key", tt.Key)
+		formValues.Set("Content-Type", tt.ContentType)
 		formValues.Set("X-Amz-Date", tt.XAmzDate)
 		formValues.Set("X-Amz-Meta-Uuid", tt.XAmzMetaUUID)
-		formValues.Set("X-Amz-Server-Side-Encryption", tt.XAmzServerSideEncryption)
 		formValues.Set("X-Amz-Algorithm", tt.XAmzAlgorithm)
 		formValues.Set("X-Amz-Credential", tt.XAmzCredential)
-		formValues.Set("Content-Type", tt.ContentType)
-		formValues.Set("Policy", tt.Policy)
-		formValues.Set("Success_action_redirect", tt.SuccessActionRedirect)
-		policyBytes, err := base64.StdEncoding.DecodeString(tt.Policy)
+		if tt.Expired {
+			// Expired already.
+			pp.SetExpires(UTCNow().AddDate(0, 0, -10))
+		} else {
+			// Expires in 10 days.
+			pp.SetExpires(UTCNow().AddDate(0, 0, 10))
+		}
+
+		formValues.Set("Policy", base64.StdEncoding.EncodeToString([]byte(pp.String())))
+		formValues.Set("Success_action_status", tt.SuccessActionStatus)
+		policyBytes, err := base64.StdEncoding.DecodeString(base64.StdEncoding.EncodeToString([]byte(pp.String())))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -81,9 +95,9 @@ func TestPostPolicyForm(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		errCode := checkPostPolicy(formValues, postPolicyForm)
-		if tt.ErrCode != errCode {
-			t.Errorf("Test %d:, Expected %d, got %d", i+1, tt.ErrCode, errCode)
+
+		if errCode := checkPostPolicy(formValues, postPolicyForm); tt.ErrCode != errCode {
+			t.Fatalf("Test %d:, Expected %d, got %d", i+1, tt.ErrCode, errCode)
 		}
 	}
 }

cmd/prepare-storage-msg.go

@@ -21,7 +21,6 @@ import (
 	"sync"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/mc/pkg/console"
 )
 
 // Helper to generate integer sequences into a friendlier user consumable format.
@@ -44,7 +43,7 @@ func printOnceFn() printOnceFunc {
 	var once sync.Once
 	return func(msg string) {
 		once.Do(func() {
-			console.Println(msg)
+			log.Println(msg)
 		})
 	}
 }
@@ -91,7 +90,7 @@ func getHealMsg(endpoints EndpointList, storageDisks []StorageAPI) string {
 
 // Prints regular message when we have sufficient disks to start the cluster.
 func printRegularMsg(endpoints EndpointList, storageDisks []StorageAPI, fn printOnceFunc) {
-	msg := getStorageInitMsg("\nInitializing data volume.", endpoints, storageDisks)
+	msg := getStorageInitMsg("Initializing data volume.", endpoints, storageDisks)
 	fn(msg)
 }
 
@@ -121,21 +120,10 @@ func getStorageInitMsg(titleMsg string, endpoints EndpointList, storageDisks []S
 
 // Prints initialization message when cluster is being initialized for the first time.
 func printFormatMsg(endpoints EndpointList, storageDisks []StorageAPI, fn printOnceFunc) {
-	msg := getStorageInitMsg("\nInitializing data volume for the first time.", endpoints, storageDisks)
+	msg := getStorageInitMsg("Initializing data volume for the first time.", endpoints, storageDisks)
 	fn(msg)
 }
 
-func printConfigErrMsg(storageDisks []StorageAPI, sErrs []error, fn printOnceFunc) {
-	msg := getConfigErrMsg(storageDisks, sErrs)
-	fn(msg)
-}
-
-// Generate a formatted message when cluster is misconfigured.
-func getConfigErrMsg(storageDisks []StorageAPI, sErrs []error) string {
-	msg := colorBlue("\nDetected configuration inconsistencies in the cluster. Please fix following servers.")
-	return msg + combineDiskErrs(storageDisks, sErrs)
-}
-
 // Combines each disk errors in a newline formatted string.
 // this is a helper function in printing messages across
 // all disks.

cmd/prepare-storage-msg_test.go

@@ -52,8 +52,6 @@ func TestHealMsg(t *testing.T) {
 		{endpoints, storageDisks, errs},
 		// Test - 2 for one of the disks is nil.
 		{endpoints, nilDisks, errs},
-		// Test - 3 for one of the errs is authentication.
-		{endpoints, nilDisks, authErrs},
 	}
 
 	for i, testCase := range testCases {
@@ -65,10 +63,6 @@ func TestHealMsg(t *testing.T) {
 		if msg == "" {
 			t.Fatalf("Test: %d Unable to get regular message.", i+1)
 		}
-		msg = getConfigErrMsg(testCase.storageDisks, testCase.serrs)
-		if msg == "" {
-			t.Fatalf("Test: %d Unable to get config error message.", i+1)
-		}
 	}
 }
 

cmd/prepare-storage.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/minio/mc/pkg/console"
 	"github.com/minio/minio/pkg/errors"
 )
 
@@ -98,23 +97,48 @@ const (
 	Abort
 )
 
+// configErrs contains the list of configuration errors.
+var configErrs = []error{
+	errInvalidAccessKeyID,
+	errAuthentication,
+	errServerVersionMismatch,
+	errServerTimeMismatch,
+}
+
 // Quick error to actions converts looking for specific errors
 // which need to be returned quickly and server should wait instead.
 func quickErrToActions(errMap map[error]int) InitActions {
 	var action InitActions
-	switch {
-	case errMap[errInvalidAccessKeyID] > 0:
-		fallthrough
-	case errMap[errAuthentication] > 0:
-		fallthrough
-	case errMap[errServerVersionMismatch] > 0:
-		fallthrough
-	case errMap[errServerTimeMismatch] > 0:
-		action = WaitForConfig
+	for _, configErr := range configErrs {
+		if errMap[configErr] > 0 {
+			action = WaitForConfig
+			break
+		}
 	}
 	return action
 }
 
+// reduceInitXLErrs reduces errors found in distributed XL initialization
+func reduceInitXLErrs(storageDisks []StorageAPI, sErrs []error) error {
+	var foundErrs int
+	for i := range sErrs {
+		if sErrs[i] != nil {
+			foundErrs++
+		}
+	}
+	if foundErrs == 0 {
+		return nil
+	}
+	// Early quit if there is a config error
+	for i := range sErrs {
+		if contains(configErrs, sErrs[i]) {
+			return fmt.Errorf("%s: %s", storageDisks[i], sErrs[i])
+		}
+	}
+	// Combine all disk errors otherwise for user inspection
+	return fmt.Errorf("%s", combineDiskErrs(storageDisks, sErrs))
+}
+
 // Preparatory initialization stage for XL validates known errors.
 // Converts them into specific actions. These actions have special purpose
 // which caller decides on what needs to be done.
@@ -262,13 +286,11 @@ func retryFormattingXLDisks(firstDisk bool, endpoints EndpointList, storageDisks
 			// Check if this is a XL or distributed XL, anything > 1 is considered XL backend.
 			switch prepForInitXL(firstDisk, sErrs, len(storageDisks)) {
 			case Abort:
-				return fmt.Errorf("%s", combineDiskErrs(storageDisks, sErrs))
+				return reduceInitXLErrs(storageDisks, sErrs)
 			case FormatDisks:
-				console.Eraseline()
 				printFormatMsg(endpoints, storageDisks, printOnceFn())
 				return initFormatXL(storageDisks)
 			case InitObjectLayer:
-				console.Eraseline()
 				// Validate formats loaded before proceeding forward.
 				err := genericFormatCheckXL(formatConfigs, sErrs)
 				if err == nil {
@@ -283,17 +305,17 @@ func retryFormattingXLDisks(firstDisk bool, endpoints EndpointList, storageDisks
 				}
 				return err
 			case WaitForQuorum:
-				console.Printf(
+				log.Printf(
 					"Initializing data volume. Waiting for minimum %d servers to come online. (elapsed %s)\n",
 					len(storageDisks)/2+1, getElapsedTime(),
 				)
 			case WaitForConfig:
 				// Print configuration errors.
-				printConfigErrMsg(storageDisks, sErrs, printOnceFn())
+				return reduceInitXLErrs(storageDisks, sErrs)
 			case WaitForAll:
-				console.Printf("Initializing data volume for first time. Waiting for other servers to come online (elapsed %s)\n", getElapsedTime())
+				log.Printf("Initializing data volume for first time. Waiting for other servers to come online (elapsed %s)\n", getElapsedTime())
 			case WaitForFormatting:
-				console.Printf("Initializing data volume for first time. Waiting for first server to come online (elapsed %s)\n", getElapsedTime())
+				log.Printf("Initializing data volume for first time. Waiting for first server to come online (elapsed %s)\n", getElapsedTime())
 			}
 		case <-globalServiceDoneCh:
 			return fmt.Errorf("Initializing data volumes gracefully stopped")

cmd/prepare-storage_test.go

@@ -16,7 +16,10 @@
 
 package cmd
 
-import "testing"
+import (
+	"os"
+	"testing"
+)
 
 func (action InitActions) String() string {
 	switch action {
@@ -41,6 +44,42 @@ func (action InitActions) String() string {
 	}
 }
 
+func TestReduceInitXLErrs(t *testing.T) {
+	_, fsDirs, err := prepareXL(4)
+	if err != nil {
+		t.Fatalf("Unable to initialize 'XL' object layer.")
+	}
+
+	// Remove all dirs.
+	for _, dir := range fsDirs {
+		defer os.RemoveAll(dir)
+	}
+
+	storageDisks, err := initStorageDisks(mustGetNewEndpointList(fsDirs...))
+	if err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
+	testCases := []struct {
+		sErrs       []error
+		expectedErr string
+	}{
+		{[]error{nil, nil, nil, nil}, ""},
+		{[]error{errUnformattedDisk, nil, nil, nil}, "\n[01/04] " + storageDisks[0].String() + " : unformatted disk found"},
+		{[]error{errUnformattedDisk, errUnformattedDisk, nil, nil}, "\n[01/04] " + storageDisks[0].String() + " : unformatted disk found" + "\n[02/04] " + storageDisks[1].String() + " : unformatted disk found"},
+		{[]error{errUnformattedDisk, errUnformattedDisk, errServerVersionMismatch, nil}, storageDisks[2].String() + ": Server versions do not match"},
+	}
+	for i, test := range testCases {
+		actual := reduceInitXLErrs(storageDisks, test.sErrs)
+		if test.expectedErr == "" && actual != nil {
+			t.Errorf("Test %d expected no error but received `%s`", i+1, actual.Error())
+		}
+		if test.expectedErr != "" && actual.Error() != test.expectedErr {
+			t.Errorf("Test %d expected `%s` but received `%s`", i+1, test.expectedErr, actual.Error())
+		}
+	}
+}
+
 func TestPrepForInitXL(t *testing.T) {
 	// All disks are unformatted, a fresh setup.
 	allUnformatted := []error{