posix: do not upstream errors in deleteFile (#4771)

This commit changes posix's deleteFile() to not upstream errors from
removing parent directories. This fixes a race condition.

The race condition occurs when multiple deleteFile()s are called on the
same parent directory, but different child files. Because deleteFile()
recursively removes parent directories if they are empty, but
deleteFile() errors if the selected deletePath does not exist, there was
an opportunity for a race condition. The two processes would remove the
child directories successfully, then depend on the parent directory
still existing. In some cases this is an invalid assumption, because
other processes can remove the parent directory beforehand. This commit
changes deleteFile() to not upstream an error if one occurs, because the
only required error should be from the immediate deletePath, not from a
parent path.

In the specific bug report, multiple CompleteMultipartUpload requests
would launch multiple deleteFile() requests. Because they chain up on
parent directories, ultimately at the end, there would be multiple
remove files for the ultimate parent directory,
.minio.sys/multipart/{bucket}. Because only one will succeed and one
will fail, an error would be upstreamed saying that the file does not
exist, and the CompleteMultipartUpload code interpreted this as
NoSuchKey, or that the object/part id doesn't exist. This was faulty
behavior and is now fixed.

The added test fails before this change and passes after this change.

Fixes: https://github.com/minio/minio/issues/4727

.github/ISSUE_TEMPLATE.md

@@ -26,8 +26,8 @@
 
 ## Your Environment
 <!--- Include as many relevant details about the environment you experienced the bug in -->
-* Version used:
+* Version used (`minio version`):
 * Environment name and version (e.g. nginx 1.9.1):
 * Server type and version:
-* Operating System and version:
+* Operating System and version (`uname -a`):
 * Link to your project:

.jshintrc

@@ -0,0 +1,4 @@
+{
+  "asi": true,
+  "esnext": true   
+}

.travis.yml

@@ -18,6 +18,8 @@ env:
 script:
 ## Run all the tests
 - make
+- diff -au <(gofmt -d cmd) <(printf "")
+- diff -au <(gofmt -d pkg) <(printf "")
 - make test GOFLAGS="-timeout 15m -race -v"
 - make coverage
 

Dockerfile

@@ -1,5 +1,7 @@
 FROM alpine:3.5
 
+MAINTAINER Minio Inc <dev@minio.io>
+
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
@@ -9,11 +11,20 @@ WORKDIR /go/src/github.com/minio/
 RUN  \
      apk add --no-cache ca-certificates && \
      apk add --no-cache --virtual .build-deps git go musl-dev && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
      rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
 
 EXPOSE 9000
-ENTRYPOINT ["minio"]
+
+COPY buildscripts/docker-entrypoint.sh /usr/bin/
+
+RUN chmod +x /usr/bin/docker-entrypoint.sh
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
 VOLUME ["/export"]
+
+CMD ["minio"]

Dockerfile.aarch64

@@ -1,5 +1,7 @@
 FROM resin/aarch64-alpine:3.5
 
+MAINTAINER Minio Inc <dev@minio.io>
+
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
@@ -9,11 +11,20 @@ WORKDIR /go/src/github.com/minio/
 RUN  \
      apk add --no-cache ca-certificates && \
      apk add --no-cache --virtual .build-deps git go musl-dev && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \     
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
      rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
 
 EXPOSE 9000
-ENTRYPOINT ["minio"]
+
+COPY buildscripts/docker-entrypoint.sh /usr/bin/
+
+RUN chmod +x /usr/bin/docker-entrypoint.sh
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
 VOLUME ["/export"]
+
+CMD ["minio"]

Dockerfile.armhf

@@ -1,5 +1,7 @@
 FROM resin/armhf-alpine:3.5
 
+MAINTAINER Minio Inc <dev@minio.io>
+
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
@@ -9,11 +11,20 @@ WORKDIR /go/src/github.com/minio/
 RUN  \
      apk add --no-cache ca-certificates && \
      apk add --no-cache --virtual .build-deps git go musl-dev && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
      rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
 
 EXPOSE 9000
-ENTRYPOINT ["minio"]
+
+COPY buildscripts/docker-entrypoint.sh /usr/bin/
+
+RUN chmod +x /usr/bin/docker-entrypoint.sh
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
 VOLUME ["/export"]
+
+CMD ["minio"]

Dockerfile.release

@@ -0,0 +1,26 @@
+FROM alpine:3.5
+
+MAINTAINER Minio Inc <dev@minio.io>
+ 
+COPY buildscripts/docker-entrypoint.sh buildscripts/healthcheck.sh /usr/bin/ 
+
+RUN \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps curl && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
+CMD ["minio"]
+

Dockerfile.release.aarch64

@@ -0,0 +1,25 @@
+FROM resin/aarch64-alpine:3.5
+
+MAINTAINER Minio Inc <dev@minio.io>
+
+COPY buildscripts/docker-entrypoint.sh buildscripts/healthcheck.sh /usr/bin/ 
+
+RUN \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps curl && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.minio.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
+CMD ["minio"]

Dockerfile.release.armhf

@@ -0,0 +1,25 @@
+FROM resin/armhf-alpine:3.5
+
+MAINTAINER Minio Inc <dev@minio.io>
+
+COPY buildscripts/docker-entrypoint.sh buildscripts/healthcheck.sh /usr/bin/ 
+
+RUN \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps curl && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.minio.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
+CMD ["minio"]

Makefile

@@ -1,100 +1,50 @@
-LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 PWD := $(shell pwd)
 GOPATH := $(shell go env GOPATH)
+LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
+
 BUILD_LDFLAGS := '$(LDFLAGS)'
-TAG := latest
 
-HOST ?= $(shell uname)
-CPU ?= $(shell uname -m)
-
-# if no host is identifed (no uname tool)
-# we assume a Linux-64bit build
-ifeq ($(HOST),)
-  HOST = Linux
-endif
-
-# identify CPU
-ifeq ($(CPU), x86_64)
-  HOST := $(HOST)64
-else
-ifeq ($(CPU), amd64)
-  HOST := $(HOST)64
-else
-ifeq ($(CPU), i686)
-  HOST := $(HOST)32
-endif
-endif
-endif
-
-
-#############################################
-# now we find out the target OS for
-# which we are going to compile in case
-# the caller didn't yet define OS himself
-ifndef (OS)
-  ifeq ($(HOST), Linux64)
-    arch = gcc
-  else
-  ifeq ($(HOST), Linux32)
-    arch = 32
-  else
-  ifeq ($(HOST), Darwin64)
-    arch = clang
-  else
-  ifeq ($(HOST), Darwin32)
-    arch = clang
-  else
-  ifeq ($(HOST), FreeBSD64)
-    arch = gcc
-  endif
-  endif
-  endif
-  endif
-  endif
-endif
-
-all: install
+all: build
 
 checks:
-	@echo "Checking deps:"
+	@echo "Check deps"
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
+	@echo "Checking project is in GOPATH"
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
 getdeps: checks
-	@echo "Installing golint:" && go get -u github.com/golang/lint/golint
-	@echo "Installing gocyclo:" && go get -u github.com/fzipp/gocyclo
-	@echo "Installing deadcode:" && go get -u github.com/remyoudompheng/go-misc/deadcode
-	@echo "Installing misspell:" && go get -u github.com/client9/misspell/cmd/misspell
-	@echo "Installing ineffassign:" && go get -u github.com/gordonklaus/ineffassign
+	@echo "Installing golint" && go get -u github.com/golang/lint/golint
+	@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
+	@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
+	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
+	@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
 
-verifiers: vet fmt lint cyclo spelling
+verifiers: getdeps vet fmt lint cyclo spelling
 
 vet:
-	@echo "Running $@:"
-	@go vet github.com/minio/minio/cmd/...
-	@go vet github.com/minio/minio/pkg/...
+	@echo "Running $@"
+	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult cmd
+	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult pkg
 
 fmt:
-	@echo "Running $@:"
-	@gofmt -s -l cmd
-	@gofmt -s -l pkg
+	@echo "Running $@"
+	@gofmt -d cmd
+	@gofmt -d pkg
 
 lint:
-	@echo "Running $@:"
+	@echo "Running $@"
 	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd...
 	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg...
 
 ineffassign:
-	@echo "Running $@:"
+	@echo "Running $@"
 	@${GOPATH}/bin/ineffassign .
 
 cyclo:
-	@echo "Running $@:"
+	@echo "Running $@"
 	@${GOPATH}/bin/gocyclo -over 100 cmd
 	@${GOPATH}/bin/gocyclo -over 100 pkg
 
-build: getdeps verifiers $(UI_ASSETS)
-
 deadcode:
 	@${GOPATH}/bin/deadcode
 
@@ -103,33 +53,43 @@ spelling:
 	@${GOPATH}/bin/misspell -error `find pkg/`
 	@${GOPATH}/bin/misspell -error `find docs/`
 
-test: build
-	@echo "Running all minio testing:"
+# Builds minio, runs the verifiers then runs the tests.
+check: test
+test: verifiers build
+	@echo "Running all minio testing"
 	@go test $(GOFLAGS) .
 	@go test $(GOFLAGS) github.com/minio/minio/cmd...
 	@go test $(GOFLAGS) github.com/minio/minio/pkg...
 
 coverage: build
-	@echo "Running all coverage for minio:"
+	@echo "Running all coverage for minio"
 	@./buildscripts/go-coverage.sh
 
-gomake-all: build
-	@echo "Installing minio:"
-	@go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
+# Builds minio locally.
+build:
+	@echo "Building minio to $(PWD)/minio ..."
+	@CGO_ENABLED=0 go build --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
 
 pkg-add:
-	${GOPATH}/bin/govendor add $(PKG)
+	@echo "Adding new package $(PKG)"
+	@${GOPATH}/bin/govendor add $(PKG)
 
 pkg-update:
-	${GOPATH}/bin/govendor update $(PKG)
+	@echo "Updating new package $(PKG)"
+	@${GOPATH}/bin/govendor update $(PKG)
 
 pkg-remove:
-	${GOPATH}/bin/govendor remove $(PKG)
+	@echo "Remove new package $(PKG)"
+	@${GOPATH}/bin/govendor remove $(PKG)
 
 pkg-list:
 	@$(GOPATH)/bin/govendor list
 
-install: gomake-all
+# Builds minio and installs it to $GOPATH/bin.
+install: build
+	@echo "Installing minio at $(GOPATH)/bin/minio ..."
+	@cp $(PWD)/minio $(GOPATH)/bin/minio
+	@echo "Check 'minio -h' for help."
 
 release: verifiers
 	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
@@ -138,7 +98,7 @@ experimental: verifiers
 	@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
 
 clean:
-	@echo "Cleaning up all the generated files:"
+	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
 	@rm -rf build
 	@rm -rf release

README.md

@@ -18,19 +18,26 @@ docker run -p 9000:9000 minio/minio:edge server /export
 ```
 Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
-## OS X
+## macOS
 ### Homebrew
 Install minio packages using [Homebrew](http://brew.sh/)
 
 ```sh
-brew install minio
+brew install minio/stable/minio
 minio server ~/Photos
 ```
+#### Note
+If you previously installed minio using `brew install minio` then reinstall minio from `minio/stable/minio` official repo. Homebrew builds are unstable due to golang 1.8 bugs.
+
+```
+brew uninstall minio 
+brew install minio/stable/minio
+```  
 
 ### Binary Download
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
-|Apple OS X|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio|
+|Apple macOS|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio |
 ```sh
 chmod 755 minio
 ./minio server ~/Photos
@@ -40,11 +47,11 @@ chmod 755 minio
 ### Binary Download
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
-|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio|
-||32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio|
-||32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio|
-||64-bit ARM|https://dl.minio.io/server/minio/release/linux-arm64/minio|
-||32-bit ARMv6|https://dl.minio.io/server/minio/release/linux-arm6vl/minio|
+|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio |
+|         |32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio |
+|         |32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio |
+|         |64-bit ARM|https://dl.minio.io/server/minio/release/linux-arm64/minio |
+|         |32-bit ARMv6|https://dl.minio.io/server/minio/release/linux-arm6vl/minio |
 ```sh
 chmod +x minio
 ./minio server ~/Photos
@@ -54,32 +61,38 @@ chmod +x minio
 ### Binary Download
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
-|Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe|
-||32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe|
+|Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe |
+|                 |32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe |
 ```sh
 minio.exe server D:\Photos
 ```
 
 ## FreeBSD
+### Port
+Install minio packages using [pkg](https://github.com/freebsd/pkg)
+
+```sh
+pkg install minio
+sysrc minio_enable=yes
+sysrc minio_disks=/home/user/Photos
+service minio start
+```
+
 ### Binary Download
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
-|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
+|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio |
 ```sh
 chmod 755 minio
 ./minio server ~/Photos
 ```
 
-You can run Minio on FreeBSD with FreeNAS storage-backend - see [here](https://docs.minio.io/docs/how-to-run-minio-in-freenas) for more details.
-
 ## Install from Source
 
 Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
 
 ```sh
-
 go get -u github.com/minio/minio
-
 ```
 
 ## Test using Minio Browser

appveyor.yml

@@ -24,6 +24,8 @@ install:
 # To run your custom scripts instead of automatic MSBuild
 build_script:
   # Compile
+  # We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
+  - ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
   - appveyor AddCompilationMessage "Starting Compile"
   - cd c:\gopath\src\github.com\minio\minio
   - go run buildscripts/gen-ldflags.go > temp.txt
@@ -36,9 +38,9 @@ test_script:
   # Unit tests
   - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
   - mkdir build\coverage
-  - go test -timeout 17m -race github.com/minio/minio/cmd...
-  - go test -race github.com/minio/minio/pkg...
-  - go test -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
+  - go test -v -timeout 17m -race github.com/minio/minio/cmd...
+  - go test -v -race github.com/minio/minio/pkg...
+  - go test -v -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
   - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
 
 after_test:

browser/.babelrc

@@ -1,8 +1,9 @@
 {
-    "presets": [
-        "es2015",
-        "react"
-    ],
-
-    "plugins": ["transform-object-rest-spread"]
-}
+  "presets": [
+    "es2015",
+    "react"
+  ],
+  "plugins": [
+    "transform-object-rest-spread"
+  ]
+}

browser/README.md

@@ -27,7 +27,7 @@ go get github.com/elazarl/go-bindata-assetfs/...
 yarn release
 ```
 
-This generates ui-assets.go in the current direcotry. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
+This generates ui-assets.go in the current directory. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
 
 ### Run Minio Browser with live reload
 

browser/app/index.js

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import 'babel-polyfill'
 
 import './less/main.less'
 
@@ -44,7 +45,6 @@ import Web from './js/web'
 window.Web = Web
 
 import storage from 'local-storage-fallback'
-
 const store = applyMiddleware(thunkMiddleware)(createStore)(reducer)
 const Browse = connect(state => state)(_Browse)
 const Login = connect(state => state)(_Login)

browser/app/js/actions.js

@@ -26,6 +26,8 @@ export const SET_BUCKETS = 'SET_BUCKETS'
 export const ADD_BUCKET = 'ADD_BUCKET'
 export const SET_VISIBLE_BUCKETS = 'SET_VISIBLE_BUCKETS'
 export const SET_OBJECTS = 'SET_OBJECTS'
+export const APPEND_OBJECTS = 'APPEND_OBJECTS'
+export const RESET_OBJECTS = 'RESET_OBJECTS'
 export const SET_STORAGE_INFO = 'SET_STORAGE_INFO'
 export const SET_SERVER_INFO = 'SET_SERVER_INFO'
 export const SHOW_MAKEBUCKET_MODAL = 'SHOW_MAKEBUCKET_MODAL'
@@ -240,15 +242,28 @@ export const setVisibleBuckets = visibleBuckets => {
   }
 }
 
-export const setObjects = (objects, marker, istruncated) => {
+const appendObjects = (objects, marker, istruncated) => {
   return {
-    type: SET_OBJECTS,
+    type: APPEND_OBJECTS,
     objects,
     marker,
     istruncated
   }
 }
 
+export const setObjects = (objects) => {
+  return {
+    type: SET_OBJECTS,
+    objects,
+  }
+}
+
+export const resetObjects = () => {
+  return {
+    type: RESET_OBJECTS
+  }
+}
+
 export const setCurrentBucket = currentBucket => {
   return {
     type: SET_CURRENT_BUCKET,
@@ -316,7 +331,7 @@ export const listObjects = () => {
           object.name = object.name.replace(`${currentPath}`, '');
           return object
         })
-        dispatch(setObjects(objects, res.nextmarker, res.istruncated))
+        dispatch(appendObjects(objects, res.nextmarker, res.istruncated))
         dispatch(setPrefixWritable(res.writable))
         dispatch(setLoadBucket(''))
         dispatch(setLoadPath(''))
@@ -337,7 +352,7 @@ export const listObjects = () => {
 export const selectPrefix = prefix => {
   return (dispatch, getState) => {
     const {currentBucket, web} = getState()
-    dispatch(setObjects([], "", false))
+    dispatch(resetObjects())
     dispatch(setLoadPath(prefix))
     web.ListObjects({
       bucketName: currentBucket,
@@ -352,7 +367,7 @@ export const selectPrefix = prefix => {
           object.name = object.name.replace(`${prefix}`, '');
           return object
         })
-        dispatch(setObjects(
+        dispatch(appendObjects(
           objects,
           res.nextmarker,
           res.istruncated
@@ -417,6 +432,8 @@ export const setLoginError = () => {
 
 export const downloadSelected = (url, req, xhr) => {
   return (dispatch) => {
+    var anchor = document.createElement('a')
+    document.body.appendChild(anchor);
     xhr.open('POST', url, true)
     xhr.responseType = 'blob'
 
@@ -424,10 +441,20 @@ export const downloadSelected = (url, req, xhr) => {
       if (this.status == 200) {
         dispatch(checkedObjectsReset())
         var blob = new Blob([this.response], {
-          type: 'application/zip'
+          type: 'octet/stream'
         })
         var blobUrl = window.URL.createObjectURL(blob);
-        window.location = blobUrl
+        var separator = req.prefix.length > 1 ? '-' : ''
+
+        anchor.href = blobUrl
+        anchor.download = req.bucketName+separator+req.prefix.slice(0, -1)+'.zip';
+
+
+
+
+        anchor.click()
+        window.URL.revokeObjectURL(blobUrl)
+        anchor.remove()
       }
     };
     xhr.send(JSON.stringify(req));

browser/app/js/components/Browse.js

@@ -68,7 +68,7 @@ export default class Browse extends React.Component {
           memory: res.MinioMemory,
           platform: res.MinioPlatform,
           runtime: res.MinioRuntime,
-          envVars: res.MinioEnvVars
+          info: res.MinioGlobalInfo
         })
         dispatch(actions.setServerInfo(serverInfo))
       })
@@ -150,7 +150,16 @@ export default class Browse extends React.Component {
       if (prefix === currentPath) return
       browserHistory.push(utils.pathJoin(currentBucket, encPrefix))
     } else {
-      window.location = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=${storage.getItem('token')}`
+      // Download the selected file.
+      web.CreateURLToken()
+        .then(res => {
+          let url = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=${res.token}`
+          window.location = url
+        })
+        .catch(err => dispatch(actions.showAlert({
+          type: 'danger',
+          message: err.message
+        })))
     }
   }
 
@@ -227,7 +236,12 @@ export default class Browse extends React.Component {
 
   removeObject() {
     const {web, dispatch, currentPath, currentBucket, deleteConfirmation, checkedObjects} = this.props
-    let objects = checkedObjects.length > 0 ? checkedObjects : [deleteConfirmation.object]
+    let objects = []
+    if (checkedObjects.length > 0) {
+      objects = checkedObjects.map(obj => `${currentPath}${obj}`)
+    } else {
+      objects = [deleteConfirmation.object]
+    }
 
     web.RemoveObject({
       bucketname: currentBucket,
@@ -365,8 +379,24 @@ export default class Browse extends React.Component {
   }
 
   handleExpireValue(targetInput, inc, object) {
-    inc === -1 ? this.refs[targetInput].stepDown(1) : this.refs[targetInput].stepUp(1)
+    let value = this.refs[targetInput].value
+    let maxValue = (targetInput == 'expireHours') ? 23 : (targetInput == 'expireMins') ? 59 : (targetInput == 'expireDays') ? 7 : 0
+    value = isNaN(value) ? 0 : value
 
+    // Use custom step count to support browser Edge
+    if((inc === -1)) {
+      if(value != 0) {
+        value--
+      }
+    }
+    else {
+      if(value != maxValue) {
+        value++
+      }
+    }
+    this.refs[targetInput].value = value
+
+    // Reset hours and mins when days reaches it's max value
     if (this.refs.expireDays.value == 7) {
       this.refs.expireHours.value = 0
       this.refs.expireMins.value = 0
@@ -374,6 +404,7 @@ export default class Browse extends React.Component {
     if (this.refs.expireDays.value + this.refs.expireHours.value + this.refs.expireMins.value == 0) {
       this.refs.expireDays.value = 7
     }
+
     const {dispatch} = this.props
     dispatch(actions.shareObject(object, this.refs.expireDays.value, this.refs.expireHours.value, this.refs.expireMins.value))
   }
@@ -384,16 +415,24 @@ export default class Browse extends React.Component {
   }
 
   downloadSelected() {
-    const {dispatch} = this.props
+    const {dispatch, web} = this.props
     let req = {
       bucketName: this.props.currentBucket,
       objects: this.props.checkedObjects,
       prefix: this.props.currentPath
     }
-    let requestUrl = location.origin + "/minio/zip?token=" + localStorage.token
 
-    this.xhr = new XMLHttpRequest()
-    dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
+    web.CreateURLToken()
+      .then(res => {
+        let requestUrl = location.origin + "/minio/zip?token=" + res.token
+
+        this.xhr = new XMLHttpRequest()
+        dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
+      })
+      .catch(err => dispatch(actions.showAlert({
+        type: 'danger',
+        message: err.message
+      })))
   }
 
   clearSelected() {
@@ -458,21 +497,25 @@ export default class Browse extends React.Component {
     }
 
     if (web.LoggedIn()) {
-      storageUsageDetails = <div className="feh-usage">
-                              <div className="fehu-chart">
-                                <div style={ { width: usedPercent } }></div>
-                              </div>
-                              <ul>
-                                <li>
-                                  Used:
-                                  { humanize.filesize(total - free) }
-                                </li>
-                                <li className="pull-right">
-                                  Free:
-                                  { humanize.filesize(total - used) }
-                                </li>
-                              </ul>
-                            </div>
+      if (!(used === 0 && free === 0)) {
+        storageUsageDetails = <div className="feh-usage">
+          <div className="fehu-chart">
+            <div style={ { width: usedPercent } }></div>
+          </div>
+          <ul>
+            <li>
+              <span>Used: </span>
+              { humanize.filesize(total - free) }
+            </li>
+            <li className="pull-right">
+              <span>Free: </span>
+              { humanize.filesize(total - used) }
+            </li>
+          </ul>
+        </div>
+      }
+
+
     }
 
     let createButton = ''
@@ -717,11 +760,11 @@ export default class Browse extends React.Component {
                 </div>
                 <div className="input-group" style={ { display: web.LoggedIn() ? 'block' : 'none' } }>
                   <label>
-                    Expires in
+                    Expires in (Max 7 days)
                   </label>
                   <div className="set-expire">
                     <div className="set-expire-item">
-        <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireDays', 1, shareObject.object) }></i>
+                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireDays', 1, shareObject.object) } />
                       <div className="set-expire-title">
                         Days
                       </div>
@@ -730,12 +773,14 @@ export default class Browse extends React.Component {
                           type="number"
                           min={ 0 }
                           max={ 7 }
-                          defaultValue={ 5 } />
+                          defaultValue={ 5 }
+                          readOnly="readOnly"
+                        />
                       </div>
-        <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireDays', -1, shareObject.object) }></i>
+                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireDays', -1, shareObject.object) } />
                     </div>
                     <div className="set-expire-item">
-        <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireHours', 1, shareObject.object) }></i>
+                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireHours', 1, shareObject.object) } />
                       <div className="set-expire-title">
                         Hours
                       </div>
@@ -744,12 +789,14 @@ export default class Browse extends React.Component {
                           type="number"
                           min={ 0 }
                           max={ 23 }
-                          defaultValue={ 0 } />
+                          defaultValue={ 0 }
+                          readOnly="readOnly"
+                        />
                       </div>
-        <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireHours', -1, shareObject.object) }></i>
+                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireHours', -1, shareObject.object) } />
                     </div>
                     <div className="set-expire-item">
-        <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireMins', 1, shareObject.object) }></i>
+                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireMins', 1, shareObject.object) } />
                       <div className="set-expire-title">
                         Minutes
                       </div>
@@ -758,11 +805,13 @@ export default class Browse extends React.Component {
                           type="number"
                           min={ 0 }
                           max={ 59 }
-                          defaultValue={ 0 } />
+                          defaultValue={ 0 }
+                          readOnly="readOnly"
+                        />
                       </div>
-        <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireMins', -1, shareObject.object) }></i>
+                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireMins', -1, shareObject.object) } />
                     </div>
-        </div>
+                  </div>
                 </div>
               </ModalBody>
               <div className="modal-footer">

browser/app/js/components/Login.js

@@ -89,10 +89,6 @@ export default class Login extends React.Component {
         { alertBox }
         <div className="l-wrap">
           <form onSubmit={ this.handleSubmit.bind(this) }>
-            <input name="fixBrowser"
-              autoComplete="username"
-              type="text"
-              style={ { display: 'none' } } />
             <InputGroup className="ig-dark"
               label="Access Key"
               id="accessKey"
@@ -102,7 +98,6 @@ export default class Login extends React.Component {
               required="required"
               autoComplete="username">
             </InputGroup>
-            <input type="text" autoComplete="new-password" style={ { display: 'none' } } />
             <InputGroup className="ig-dark"
               label="Secret Key"
               id="secretKey"

browser/app/js/components/PolicyInput.js

@@ -7,6 +7,7 @@ import * as actions from '../actions'
 class PolicyInput extends Component {
   componentDidMount() {
     const {web, dispatch} = this.props
+    this.prefix.focus()
     web.ListAllBucketPolicies({
       bucketName: this.props.currentBucket
     }).then(res => {
@@ -27,8 +28,23 @@ class PolicyInput extends Component {
 
   handlePolicySubmit(e) {
     e.preventDefault()
-    const {web, dispatch} = this.props
+    const {web, dispatch, currentBucket} = this.props
 
+    let prefix = currentBucket + '/' + this.prefix.value
+    let policy = this.policy.value
+
+    if (!prefix.endsWith('*')) prefix = prefix + '*'
+ 
+    let prefixAlreadyExists = this.props.policies.some(elem => prefix === elem.prefix)
+
+    if (prefixAlreadyExists) {
+      dispatch(actions.showAlert({
+       type: 'danger',
+       message: "Policy for this prefix already exists."
+      }))
+      return
+    }
+    
     web.SetBucketPolicy({
       bucketName: this.props.currentBucket,
       prefix: this.prefix.value,
@@ -36,8 +52,7 @@ class PolicyInput extends Component {
     })
       .then(() => {
         dispatch(actions.setPolicies([{
-          policy: this.policy.value,
-          prefix: this.prefix.value + '*',
+          policy, prefix
         }, ...this.props.policies]))
         this.prefix.value = ''
       })

browser/app/js/components/SettingsModal.js

@@ -34,23 +34,12 @@ class SettingsModal extends React.Component {
 
     let accessKeyEnv = ''
     let secretKeyEnv = ''
-    // Check environment variables first. They may or may not have been
-    // loaded already; they load in Browse#componentDidMount.
-    if (serverInfo.envVars) {
-      serverInfo.envVars.forEach(envVar => {
-        let keyVal = envVar.split('=')
-        if (keyVal[0] == 'MINIO_ACCESS_KEY') {
-          accessKeyEnv = keyVal[1]
-        } else if (keyVal[0] == 'MINIO_SECRET_KEY') {
-          secretKeyEnv = keyVal[1]
-        }
-      })
-    }
-    if (accessKeyEnv != '' || secretKeyEnv != '') {
+    // Check environment variables first.
+    if (serverInfo.info.isEnvCreds) {
       dispatch(actions.setSettings({
-        accessKey: accessKeyEnv,
-        secretKey: secretKeyEnv,
-        keysReadOnly: true
+          accessKey: 'xxxxxxxxx',
+          secretKey: 'xxxxxxxxx',
+          keysReadOnly: true
       }))
     } else {
       web.GetAuth()

browser/app/js/reducers.js

@@ -77,16 +77,18 @@ export default (state = {
     case actions.SET_CURRENT_BUCKET:
       newState.currentBucket = action.currentBucket
       break
+    case actions.APPEND_OBJECTS:
+      newState.objects = [...newState.objects, ...action.objects]
+      newState.marker = action.marker
+      newState.istruncated = action.istruncated
+      break
     case actions.SET_OBJECTS:
-      if (!action.objects.length) {
-        newState.objects = []
-        newState.marker = ""
-        newState.istruncated = action.istruncated
-      } else {
-        newState.objects = [...action.objects]
-        newState.marker = action.marker
-        newState.istruncated = action.istruncated
-      }
+      newState.objects = [...action.objects]
+      break
+    case actions.RESET_OBJECTS:
+      newState.objects = []
+      newState.marker = ""
+      newState.istruncated = false
       break
     case actions.SET_CURRENT_PATH:
       newState.currentPath = action.currentPath

browser/app/js/web.js

@@ -112,6 +112,9 @@ export default class Web {
         return res
       })
   }
+  CreateURLToken() {
+    return this.makeCall('CreateURLToken')
+  }
   GetBucketPolicy(args) {
     return this.makeCall('GetBucketPolicy', args)
   }

browser/app/less/inc/form.less

@@ -81,7 +81,7 @@ select.form-control {
     width: 100%;
     height: 40px;
     border: 0;
-    background: transparent;
+    background: transparent !important;
     text-align: center;
     position: relative;
     z-index: 1;
@@ -114,8 +114,8 @@ select.form-control {
 
 .ig-dark {
     .ig-text {
-        color: @white;
-        border-color: rgba(255,255,255,0.1);
+        color: @white !important;
+        border-color: rgba(255,255,255,0.1) !important;
     }
 
     .ig-helpers {
@@ -183,6 +183,17 @@ select.form-control {
 .set-expire {
     border: 1px solid @input-border;
     margin: 35px 0 30px;
+    position: relative;
+
+    &:before {
+        content: '';
+        position: absolute;
+        left: 0;
+        top: 0;
+        width: 100%;
+        height: 100%;
+        z-index: 1;
+    }
 }
 
 .set-expire-item {
@@ -191,6 +202,7 @@ select.form-control {
     display: table-cell;
     width: 1%;
     text-align: center;
+    .user-select(none);
 
     &:not(:last-child) {
         border-right: 1px solid @input-border;
@@ -209,6 +221,7 @@ select.form-control {
     left: -8px;
 
     input {
+        .user-select(none);
         font-size: 20px;
         text-align: center;
         position: relative;

browser/webpack.config.js

@@ -22,7 +22,6 @@ var purify = require("purifycss-webpack-plugin")
 var exports = {
   context: __dirname,
   entry: [
-    "babel-polyfill",
     path.resolve(__dirname, 'app/index.js')
   ],
   output: {
@@ -101,7 +100,6 @@ var exports = {
 
 if (process.env.NODE_ENV === 'dev') {
   exports.entry = [
-    "babel-polyfill",
     'webpack/hot/dev-server',
     'webpack-dev-server/client?http://localhost:8080',
     path.resolve(__dirname, 'app/index.js')

buildscripts/build.sh

@@ -5,7 +5,7 @@ _init() {
     LDFLAGS=$(go run buildscripts/gen-ldflags.go)
 
     # Extract release tag
-    release_tag=$(echo $LDFLAGS | awk {'print $4'} | cut -f2 -d=)
+    release_tag=$(echo $LDFLAGS | awk {'print $6'} | cut -f2 -d=)
 
     # Verify release tag.
     if [ -z "$release_tag" ]; then
@@ -28,6 +28,7 @@ _init() {
     ## System binaries
     CP=`which cp`
     SHASUM=`which shasum`
+    SHA256SUM="${SHASUM} -a 256"
     SED=`which sed`
 }
 
@@ -43,58 +44,36 @@ go_build() {
     release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
     # Release binary downloadable name
     release_real_bin="$release_str/$os-$arch/$(basename $package)"
-    # Release shasum name
-    release_shasum="$release_str/$os-$arch/$(basename $package).shasum"
+
+    # Release sha1sum name
+    release_shasum="$release_str/$os-$arch/$(basename $package).${release_tag}.shasum"
+    # Release sha1sum default
+    release_shasum_default="$release_str/$os-$arch/$(basename $package).shasum"
+
+    # Release sha256sum name
+    release_sha256sum="$release_str/$os-$arch/$(basename $package).${release_tag}.sha256sum"
+    # Release sha256sum default
+    release_sha256sum_default="$release_str/$os-$arch/$(basename $package).sha256sum"
 
     # Go build to build the binary.
-    if [ "${arch}" == "arm" ]; then
-        # Release binary downloadable name
-        release_real_bin_6="$release_str/$os-${arch}6vl/$(basename $package)"
+    CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
 
-        release_bin_6="$release_str/$os-${arch}6vl/$(basename $package).$release_tag"
-        ## Support building for ARM6vl
-        GOARM=6 GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin_6
-
-        ## Copy
-        $CP -p $release_bin_6 $release_real_bin_6
-
-        # Release shasum name
-        release_shasum_6="$release_str/$os-${arch}6vl/$(basename $package).shasum"
-
-        # Calculate shasum
-        shasum_str=$(${SHASUM} ${release_bin_6})
-        echo ${shasum_str} | $SED "s/$release_str\/$os-${arch}6vl\///g" > $release_shasum_6
-
-        # Release binary downloadable name
-        release_real_bin_7="$release_str/$os-$arch/$(basename $package)"
-
-        release_bin_7="$release_str/$os-$arch/$(basename $package).$release_tag"
-        ## Support building for ARM7vl
-        GOARM=7 GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin_7
-
-        ## Copy
-        $CP -p $release_bin_7 $release_real_bin_7
-
-        # Release shasum name
-        release_shasum_7="$release_str/$os-$arch/$(basename $package).shasum"
-
-        # Calculate shasum
-        shasum_str=$(${SHASUM} ${release_bin_7})
-        echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum_7
+    # Create copy
+    if [ $os == "windows" ]; then
+        $CP -p $release_bin ${release_real_bin}.exe
     else
-        GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
-
-        # Create copy
-        if [ $os == "windows" ]; then
-            $CP -p $release_bin ${release_real_bin}.exe
-        else
-            $CP -p $release_bin $release_real_bin
-        fi
-
-        # Calculate shasum
-        shasum_str=$(${SHASUM} ${release_bin})
-        echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
+        $CP -p $release_bin $release_real_bin
     fi
+
+    # Calculate sha1sum
+    shasum_str=$(${SHASUM} ${release_bin})
+    echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
+    $CP -p $release_shasum $release_shasum_default
+
+    # Calculate sha256sum
+    sha256sum_str=$(${SHA256SUM} ${release_bin})
+    echo ${sha256sum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_sha256sum
+    $CP -p $release_sha256sum $release_sha256sum_default
 }
 
 main() {
@@ -111,9 +90,18 @@ main() {
             go_build ${each_osarch}
         done
     else
-        for each_osarch in $(echo $chosen_osarch | sed 's/,/ /g'); do
-            go_build ${each_osarch}
+        local found=0
+        for each_osarch in ${SUPPORTED_OSARCH}; do
+            if [ "$chosen_osarch" = "$each_osarch" ]; then
+                found=1
+            fi
         done
+        if [ ${found} -eq 1 ]; then
+            go_build ${chosen_osarch}
+        else
+            echo "Unknown architecture \"${chosen_osarch}\""
+            exit 1
+        fi
     fi
 
 }

buildscripts/checkdeps.sh

@@ -25,6 +25,11 @@ _init() {
     OSX_VERSION="10.8"
     KNAME=$(uname -s)
     ARCH=$(uname -m)
+    case "${KNAME}" in
+        SunOS )
+            ARCH=$(isainfo -k)
+            ;;
+    esac
 }
 
 ## FIXME:
@@ -97,7 +102,7 @@ assert_is_supported_arch() {
 
 assert_is_supported_os() {
     case "${KNAME}" in
-        Linux | FreeBSD | OpenBSD | NetBSD | DragonFly )
+        Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS )
             return
             ;;
         Darwin )

buildscripts/checkgopath.sh

@@ -16,8 +16,6 @@
 #
 
 main() {
-    echo "Checking project is in GOPATH:"
-
     IFS=':' read -r -a paths <<< "$GOPATH"
     for path in "${paths[@]}"; do
         minio_path="$path/src/github.com/minio/minio"

buildscripts/docker-entrypoint.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+#
+# Minio Cloud Storage, (C) 2017 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# If command starts with an option, prepend minio.
+if [ "${1}" != "minio" ]; then
+    if [ -n "${1}" ]; then
+        set -- minio "$@"
+    fi
+fi
+
+# Wait for all the hosts to come online and have
+# their DNS entries populated properly.
+docker_wait_hosts() {
+    hosts="$@"
+    num_hosts=0
+    # Count number of hosts in arguments.
+    for host in $hosts; do
+        [ $(echo "$host" | grep -E "^http") ] || continue
+        num_hosts=$((num_hosts+1))
+    done
+    if [ $num_hosts -gt 0 ]; then
+        echo -n "Waiting for all hosts to resolve..."
+        while true; do
+            x=0
+            for host in $hosts; do
+                [ $(echo "$host" | grep -E "^http") ] || continue
+                # Extract the domain.
+                host=$(echo $host | sed -e 's/^http[s]\?:\/\/\([^\/]\+\).*/\1/')
+                echo -n .
+                val=$(ping -c 1 $host 2>/dev/null)
+                if [ $? != 0 ]; then
+                    echo "Failed to lookup $host"
+                    continue
+                fi
+                x=$((x+1))
+            done
+            # Provided hosts same as successful hosts, should break out.
+            test $x -eq $num_hosts && break
+            echo "Failed to resolve hosts.. retrying after 1 second."
+            sleep 1
+        done
+        echo "All hosts are resolving proceeding to initialize Minio."
+    fi
+}
+
+## Look for docker secrets in default documented location.
+docker_secrets_env() {
+    local MINIO_ACCESS_KEY_FILE="/run/secrets/access_key"
+    local MINIO_SECRET_KEY_FILE="/run/secrets/secret_key"
+
+    if [ -f $MINIO_ACCESS_KEY_FILE -a -f $MINIO_SECRET_KEY_FILE ]; then
+        if [ -f $MINIO_ACCESS_KEY_FILE ]; then
+            export MINIO_ACCESS_KEY="$(cat "$MINIO_ACCESS_KEY_FILE")"
+        fi
+        if [ -f $MINIO_SECRET_KEY_FILE ]; then
+            export MINIO_SECRET_KEY="$(cat "$MINIO_SECRET_KEY_FILE")"
+        fi
+    fi
+}
+
+## Set access env from secrets if necessary.
+docker_secrets_env
+
+## Wait for all the hosts to come online.
+docker_wait_hosts "$@"
+
+exec "$@"

buildscripts/gen-ldflags.go

@@ -27,8 +27,8 @@ import (
 )
 
 func genLDFlags(version string) string {
-	var ldflagsStr string
-	ldflagsStr = "-X github.com/minio/minio/cmd.Version=" + version
+	ldflagsStr := "-s -w"
+	ldflagsStr += " -X github.com/minio/minio/cmd.Version=" + version
 	ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag(version)
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]

buildscripts/healthcheck.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# Minio Cloud Storage, (C) 2017 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+_init () {
+    scheme="http://"
+    address="127.0.0.1:9000"
+    resource="/minio/index.html"
+}
+
+HealthCheckMain () {
+    # Get the http response code
+    http_response=$(curl -s -k -o /dev/null -I -w "%{http_code}" ${scheme}${address}${resource})
+
+    # Get the http response body
+    http_response_body=$(curl -k -s ${scheme}${address}${resource})
+
+    # server returns response 403 and body "SSL required" if non-TLS connection is attempted on a TLS-configured server. 
+    # change the scheme and try again
+    if [ "$http_response" = "403" ] && [ "$http_response_body" = "SSL required" ]; then
+        scheme="https://"
+        http_response=$(curl -s -k -o /dev/null -I -w "%{http_code}" ${scheme}${address}${resource})
+    fi
+
+    # If http_repsonse is 200 - server is up. 
+    # When MINIO_BROWSER is set to off, curl responds with 404. We assume that the the server is up
+    [ "$http_response" = "200" ] || [ "$http_response" = "404" ]
+}
+
+_init && HealthCheckMain

cmd/admin-handlers.go

@@ -24,7 +24,9 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"path"
 	"strconv"
+	"sync"
 	"time"
 )
 
@@ -49,6 +51,7 @@ const (
 	mgmtDryRun         mgmtQueryKey = "dry-run"
 	mgmtUploadIDMarker mgmtQueryKey = "upload-id-marker"
 	mgmtMaxUploads     mgmtQueryKey = "max-uploads"
+	mgmtUploadID       mgmtQueryKey = "upload-id"
 )
 
 // ServerVersion - server version
@@ -205,14 +208,45 @@ type ServerConnStats struct {
 	Throughput       uint64 `json:"throughput,omitempty"`
 }
 
-// ServerInfo holds the information that will be returned by ServerInfo API
-type ServerInfo struct {
+// ServerHTTPMethodStats holds total number of HTTP operations from/to the server,
+// including the average duration the call was spent.
+type ServerHTTPMethodStats struct {
+	Count       uint64 `json:"count"`
+	AvgDuration string `json:"avgDuration"`
+}
+
+// ServerHTTPStats holds all type of http operations performed to/from the server
+// including their average execution time.
+type ServerHTTPStats struct {
+	TotalHEADStats     ServerHTTPMethodStats `json:"totalHEADs"`
+	SuccessHEADStats   ServerHTTPMethodStats `json:"successHEADs"`
+	TotalGETStats      ServerHTTPMethodStats `json:"totalGETs"`
+	SuccessGETStats    ServerHTTPMethodStats `json:"successGETs"`
+	TotalPUTStats      ServerHTTPMethodStats `json:"totalPUTs"`
+	SuccessPUTStats    ServerHTTPMethodStats `json:"successPUTs"`
+	TotalPOSTStats     ServerHTTPMethodStats `json:"totalPOSTs"`
+	SuccessPOSTStats   ServerHTTPMethodStats `json:"successPOSTs"`
+	TotalDELETEStats   ServerHTTPMethodStats `json:"totalDELETEs"`
+	SuccessDELETEStats ServerHTTPMethodStats `json:"successDELETEs"`
+}
+
+// ServerInfoData holds storage, connections and other
+// information of a given server.
+type ServerInfoData struct {
 	StorageInfo StorageInfo      `json:"storage"`
 	ConnStats   ServerConnStats  `json:"network"`
+	HTTPStats   ServerHTTPStats  `json:"http"`
 	Properties  ServerProperties `json:"server"`
 }
 
-// ServerInfoHandler - GET /?server-info
+// ServerInfo holds server information result of one node
+type ServerInfo struct {
+	Error string          `json:"error"`
+	Addr  string          `json:"addr"`
+	Data  *ServerInfoData `json:"data"`
+}
+
+// ServerInfoHandler - GET /?info
 // ----------
 // Get server information
 func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
@@ -223,58 +257,43 @@ func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
-		return
-	}
-	storage := objLayer.StorageInfo()
+	// Web service response
+	reply := make([]ServerInfo, len(globalAdminPeers))
 
-	// Build list of enabled ARNs queues
-	var arns []string
-	for queueArn := range globalEventNotifier.GetAllExternalTargets() {
-		arns = append(arns, queueArn)
+	var wg sync.WaitGroup
+
+	// Gather server information for all nodes
+	for i, p := range globalAdminPeers {
+		wg.Add(1)
+
+		// Gather information from a peer in a goroutine
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+
+			// Initialize server info at index
+			reply[idx] = ServerInfo{Addr: peer.addr}
+
+			serverInfoData, err := peer.cmdRunner.ServerInfoData()
+			if err != nil {
+				errorIf(err, "Unable to get server info from %s.", peer.addr)
+				reply[idx].Error = err.Error()
+				return
+			}
+
+			reply[idx].Data = &serverInfoData
+		}(i, p)
 	}
 
-	// Fetch uptimes from all peers. This may fail to due to lack
-	// of read-quorum availability.
-	uptime, err := getPeerUptimes(globalAdminPeers)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		errorIf(err, "Unable to get uptime from majority of servers.")
-		return
-	}
-
-	// Build server properties information
-	properties := ServerProperties{
-		Version:  Version,
-		CommitID: CommitID,
-		Region:   serverConfig.GetRegion(),
-		SQSARN:   arns,
-		Uptime:   uptime,
-	}
-
-	// Build network info
-	connStats := ServerConnStats{
-		TotalInputBytes:  globalConnStats.getTotalInputBytes(),
-		TotalOutputBytes: globalConnStats.getTotalOutputBytes(),
-	}
-
-	// Build the whole returned information
-	info := ServerInfo{
-		StorageInfo: storage,
-		ConnStats:   connStats,
-		Properties:  properties,
-	}
+	wg.Wait()
 
 	// Marshal API response
-	jsonBytes, err := json.Marshal(info)
+	jsonBytes, err := json.Marshal(reply)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		errorIf(err, "Failed to marshal storage info into json.")
 		return
 	}
+
 	// Reply with storage information (across nodes in a
 	// distributed setup) as json.
 	writeSuccessResponseJSON(w, jsonBytes)
@@ -602,6 +621,42 @@ func isDryRun(qval url.Values) bool {
 	return false
 }
 
+// healResult - represents result of a heal operation like
+// heal-object, heal-upload.
+type healResult struct {
+	State healState `json:"state"`
+}
+
+// healState - different states of heal operation
+type healState int
+
+const (
+	// healNone - none of the disks healed
+	healNone healState = iota
+	// healPartial - some disks were healed, others were offline
+	healPartial
+	// healOK - all disks were healed
+	healOK
+)
+
+// newHealResult - returns healResult given number of disks healed and
+// number of disks offline
+func newHealResult(numHealedDisks, numOfflineDisks int) healResult {
+	var state healState
+	switch {
+	case numHealedDisks == 0:
+		state = healNone
+
+	case numOfflineDisks > 0:
+		state = healPartial
+
+	default:
+		state = healOK
+	}
+
+	return healResult{State: state}
+}
+
 // HealObjectHandler - POST /?heal&bucket=mybucket&object=myobject&dry-run
 // - x-minio-operation = object
 // - bucket and object are both mandatory query parameters
@@ -644,14 +699,95 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	err := objLayer.HealObject(bucket, object)
+	numOfflineDisks, numHealedDisks, err := objLayer.HealObject(bucket, object)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	jsonBytes, err := json.Marshal(newHealResult(numHealedDisks, numOfflineDisks))
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Return 200 on success.
-	writeSuccessResponseHeadersOnly(w)
+	writeSuccessResponseJSON(w, jsonBytes)
+}
+
+// HealUploadHandler - POST /?heal&bucket=mybucket&object=myobject&upload-id=myuploadID&dry-run
+// - x-minio-operation = upload
+// - bucket, object and upload-id are mandatory query parameters
+// Heal a given upload, if present.
+func (adminAPI adminAPIHandlers) HealUploadHandler(w http.ResponseWriter, r *http.Request) {
+	// Get object layer instance.
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, adminAPIErr, r.URL)
+		return
+	}
+
+	vars := r.URL.Query()
+	bucket := vars.Get(string(mgmtBucket))
+	object := vars.Get(string(mgmtObject))
+	uploadID := vars.Get(string(mgmtUploadID))
+	uploadObj := path.Join(bucket, object, uploadID)
+
+	// Validate bucket and object names as supplied via query
+	// parameters.
+	if err := checkBucketAndObjectNames(bucket, object); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Validate the bucket and object w.r.t backend representation
+	// of an upload.
+	if err := checkBucketAndObjectNames(minioMetaMultipartBucket,
+		uploadObj); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Check if upload exists.
+	if _, err := objLayer.GetObjectInfo(minioMetaMultipartBucket,
+		uploadObj); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// if dry-run is set in query params then perform validations
+	// and return success.
+	if isDryRun(vars) {
+		writeSuccessResponseHeadersOnly(w)
+		return
+	}
+
+	//We are able to use HealObject for healing an upload since an
+	//ongoing upload has the same backend representation as an
+	//object.  The 'object' corresponding to a given bucket,
+	//object and uploadID is
+	//.minio.sys/multipart/bucket/object/uploadID.
+	numOfflineDisks, numHealedDisks, err := objLayer.HealObject(minioMetaMultipartBucket, uploadObj)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	jsonBytes, err := json.Marshal(newHealResult(numHealedDisks, numOfflineDisks))
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Return 200 on success.
+	writeSuccessResponseJSON(w, jsonBytes)
 }
 
 // HealFormatHandler - POST /?heal&dry-run

cmd/admin-handlers_test.go

@@ -21,10 +21,12 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"reflect"
 	"testing"
 	"time"
 
@@ -154,14 +156,9 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	}
 
 	// Initialize boot time
-	globalBootTime = time.Now().UTC()
+	globalBootTime = UTCNow()
 
-	// Set globalEndpoints for a single node XL setup.
-	for _, xlDir := range xlDirs {
-		globalEndpoints = append(globalEndpoints, &url.URL{
-			Path: xlDir,
-		})
-	}
+	globalEndpoints = mustGetNewEndpointList(xlDirs...)
 
 	// Set globalIsXL to indicate that the setup uses an erasure code backend.
 	globalIsXL = true
@@ -299,14 +296,8 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	// Initialize admin peers to make admin RPC calls. Note: In a
 	// single node setup, this degenerates to a simple function
 	// call under the hood.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-
-	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
-	globalMinioAddr = eps[0].Host
-	initGlobalAdminPeers(eps)
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Setting up a go routine to simulate ServerMux's
 	// handleServiceSignals for stop and restart commands.
@@ -365,14 +356,8 @@ func TestServiceSetCreds(t *testing.T) {
 	// Initialize admin peers to make admin RPC calls. Note: In a
 	// single node setup, this degenerates to a simple function
 	// call under the hood.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-
-	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
-	globalMinioAddr = eps[0].Host
-	initGlobalAdminPeers(eps)
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	credentials := serverConfig.GetCredential()
 	var body []byte
@@ -453,14 +438,8 @@ func TestListLocksHandler(t *testing.T) {
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-
-	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
-	globalMinioAddr = eps[0].Host
-	initGlobalAdminPeers(eps)
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	testCases := []struct {
 		bucket         string
@@ -528,11 +507,7 @@ func TestClearLocksHandler(t *testing.T) {
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-	initGlobalAdminPeers(eps)
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	testCases := []struct {
 		bucket         string
@@ -756,7 +731,7 @@ func TestListObjectsHealHandler(t *testing.T) {
 	}
 	defer adminTestBed.TearDown()
 
-	err = adminTestBed.objLayer.MakeBucket("mybucket")
+	err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}
@@ -884,7 +859,7 @@ func TestHealBucketHandler(t *testing.T) {
 	}
 	defer adminTestBed.TearDown()
 
-	err = adminTestBed.objLayer.MakeBucket("mybucket")
+	err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}
@@ -961,7 +936,7 @@ func TestHealObjectHandler(t *testing.T) {
 	// Create an object myobject under bucket mybucket.
 	bucketName := "mybucket"
 	objName := "myobject"
-	err = adminTestBed.objLayer.MakeBucket(bucketName)
+	err = adminTestBed.objLayer.MakeBucketWithLocation(bucketName, "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
 	}
@@ -1048,6 +1023,160 @@ func TestHealObjectHandler(t *testing.T) {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
 		}
 	}
+
+}
+
+// buildAdminRequest - helper function to build an admin API request.
+func buildAdminRequest(queryVal url.Values, opHdr, method string,
+	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {
+	req, err := newTestRequest(method, "/?"+queryVal.Encode(), contentLength, bodySeeker)
+	if err != nil {
+		return nil, traceError(err)
+	}
+
+	req.Header.Set(minioAdminOpHeader, opHdr)
+
+	cred := serverConfig.GetCredential()
+	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
+	if err != nil {
+		return nil, traceError(err)
+	}
+
+	return req, nil
+}
+
+// mkHealUploadQuery - helper to build HealUploadHandler query string.
+func mkHealUploadQuery(bucket, object, uploadID, dryRun string) url.Values {
+	queryVal := url.Values{}
+	queryVal.Set(string(mgmtBucket), bucket)
+	queryVal.Set(string(mgmtObject), object)
+	queryVal.Set(string(mgmtUploadID), uploadID)
+	queryVal.Set("heal", "")
+	queryVal.Set(string(mgmtDryRun), dryRun)
+	return queryVal
+}
+
+// TestHealUploadHandler - test for HealUploadHandler.
+func TestHealUploadHandler(t *testing.T) {
+	adminTestBed, err := prepareAdminXLTestBed()
+	if err != nil {
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
+	}
+	defer adminTestBed.TearDown()
+
+	// Create an object myobject under bucket mybucket.
+	bucketName := "mybucket"
+	objName := "myobject"
+	err = adminTestBed.objLayer.MakeBucketWithLocation(bucketName, "")
+	if err != nil {
+		t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
+	}
+
+	// Create a new multipart upload.
+	uploadID, err := adminTestBed.objLayer.NewMultipartUpload(bucketName, objName, nil)
+	if err != nil {
+		t.Fatalf("Failed to create a new multipart upload %s/%s - %v",
+			bucketName, objName, err)
+	}
+
+	// Upload a part.
+	partID := 1
+	_, err = adminTestBed.objLayer.PutObjectPart(bucketName, objName, uploadID,
+		partID, int64(len("hello")), bytes.NewReader([]byte("hello")), "", "")
+	if err != nil {
+		t.Fatalf("Failed to upload part %d of %s/%s - %v", partID,
+			bucketName, objName, err)
+	}
+
+	testCases := []struct {
+		bucket     string
+		object     string
+		dryrun     string
+		statusCode int
+	}{
+		// 1. Valid test case.
+		{
+			bucket:     bucketName,
+			object:     objName,
+			statusCode: http.StatusOK,
+		},
+		// 2. Invalid bucket name.
+		{
+			bucket:     `invalid\\Bucket`,
+			object:     "myobject",
+			statusCode: http.StatusBadRequest,
+		},
+		// 3. Bucket not found.
+		{
+			bucket:     "bucketnotfound",
+			object:     "myobject",
+			statusCode: http.StatusNotFound,
+		},
+		// 4. Invalid object name.
+		{
+			bucket:     bucketName,
+			object:     `invalid\\Object`,
+			statusCode: http.StatusBadRequest,
+		},
+		// 5. Object not found.
+		{
+			bucket:     bucketName,
+			object:     "objectnotfound",
+			statusCode: http.StatusNotFound,
+		},
+		// 6. Valid test case with dry-run.
+		{
+			bucket:     bucketName,
+			object:     objName,
+			dryrun:     "yes",
+			statusCode: http.StatusOK,
+		},
+	}
+	for i, test := range testCases {
+		// Prepare query params.
+		queryVal := mkHealUploadQuery(test.bucket, test.object, uploadID, test.dryrun)
+		req, err1 := buildAdminRequest(queryVal, "upload", http.MethodPost, 0, nil)
+		if err1 != nil {
+			t.Fatalf("Test %d - Failed to construct heal object request - %v", i+1, err1)
+		}
+		rec := httptest.NewRecorder()
+		adminTestBed.mux.ServeHTTP(rec, req)
+		if test.statusCode != rec.Code {
+			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
+		}
+	}
+
+	sample := testCases[0]
+	// Modify authorization header after signing to test signature
+	// mismatch handling.
+	queryVal := mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun)
+	req, err := buildAdminRequest(queryVal, "upload", "POST", 0, nil)
+	if err != nil {
+		t.Fatalf("Failed to construct heal object request - %v", err)
+	}
+
+	// Set x-amz-date to a date different than time of signing.
+	req.Header.Set("x-amz-date", time.Time{}.Format(iso8601Format))
+
+	rec := httptest.NewRecorder()
+	adminTestBed.mux.ServeHTTP(rec, req)
+	if rec.Code != http.StatusForbidden {
+		t.Errorf("Expected %d but received %d", http.StatusBadRequest, rec.Code)
+	}
+
+	// Set objectAPI to nil to test Server not initialized case.
+	resetGlobalObjectAPI()
+	queryVal = mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun)
+	req, err = buildAdminRequest(queryVal, "upload", "POST", 0, nil)
+	if err != nil {
+		t.Fatalf("Failed to construct heal object request - %v", err)
+	}
+
+	rec = httptest.NewRecorder()
+	adminTestBed.mux.ServeHTTP(rec, req)
+	if rec.Code != http.StatusServiceUnavailable {
+		t.Errorf("Expected %d but received %d", http.StatusServiceUnavailable, rec.Code)
+	}
 }
 
 // TestHealFormatHandler - test for HealFormatHandler.
@@ -1061,21 +1190,11 @@ func TestHealFormatHandler(t *testing.T) {
 	// Prepare query params for heal-format mgmt REST API.
 	queryVal := url.Values{}
 	queryVal.Set("heal", "")
-	req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
+	req, err := buildAdminRequest(queryVal, "format", "POST", 0, nil)
 	if err != nil {
 		t.Fatalf("Failed to construct heal object request - %v", err)
 	}
 
-	// Set x-minio-operation header to format.
-	req.Header.Set(minioAdminOpHeader, "format")
-
-	// Sign the request using signature v4.
-	cred := serverConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign heal object request - %v", err)
-	}
-
 	rec := httptest.NewRecorder()
 	adminTestBed.mux.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
@@ -1092,34 +1211,18 @@ func TestGetConfigHandler(t *testing.T) {
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-
-	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
-	globalMinioAddr = eps[0].Host
-	initGlobalAdminPeers(eps)
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for get-config mgmt REST API.
 	queryVal := url.Values{}
 	queryVal.Set("config", "")
 
-	req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil)
+	req, err := buildAdminRequest(queryVal, "get", http.MethodGet, 0, nil)
 	if err != nil {
 		t.Fatalf("Failed to construct get-config object request - %v", err)
 	}
 
-	// Set x-minio-operation header to get.
-	req.Header.Set(minioAdminOpHeader, "get")
-
-	// Sign the request using signature v4.
-	cred := serverConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign heal object request - %v", err)
-	}
-
 	rec := httptest.NewRecorder()
 	adminTestBed.mux.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
@@ -1137,14 +1240,8 @@ func TestSetConfigHandler(t *testing.T) {
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-
-	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
-	globalMinioAddr = eps[0].Host
-	initGlobalAdminPeers(eps)
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// SetConfigHandler restarts minio setup - need to start a
 	// signal receiver to receive on globalServiceSignalCh.
@@ -1154,21 +1251,12 @@ func TestSetConfigHandler(t *testing.T) {
 	queryVal := url.Values{}
 	queryVal.Set("config", "")
 
-	req, err := newTestRequest("PUT", "/?"+queryVal.Encode(), int64(len(configJSON)), bytes.NewReader(configJSON))
+	req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(configJSON)),
+		bytes.NewReader(configJSON))
 	if err != nil {
 		t.Fatalf("Failed to construct get-config object request - %v", err)
 	}
 
-	// Set x-minio-operation header to set.
-	req.Header.Set(minioAdminOpHeader, "set")
-
-	// Sign the request using signature v4.
-	cred := serverConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign heal object request - %v", err)
-	}
-
 	rec := httptest.NewRecorder()
 	adminTestBed.mux.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
@@ -1186,6 +1274,58 @@ func TestSetConfigHandler(t *testing.T) {
 	}
 }
 
+func TestAdminServerInfo(t *testing.T) {
+	adminTestBed, err := prepareAdminXLTestBed()
+	if err != nil {
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
+	}
+	defer adminTestBed.TearDown()
+
+	// Initialize admin peers to make admin RPC calls.
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
+
+	// Prepare query params for set-config mgmt REST API.
+	queryVal := url.Values{}
+	queryVal.Set("info", "")
+
+	req, err := buildAdminRequest(queryVal, "", http.MethodGet, 0, nil)
+	if err != nil {
+		t.Fatalf("Failed to construct get-config object request - %v", err)
+	}
+
+	rec := httptest.NewRecorder()
+	adminTestBed.mux.ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Errorf("Expected to succeed but failed with %d", rec.Code)
+	}
+
+	results := []ServerInfo{}
+	err = json.NewDecoder(rec.Body).Decode(&results)
+	if err != nil {
+		t.Fatalf("Failed to decode set config result json %v", err)
+	}
+
+	if len(results) == 0 {
+		t.Error("Expected at least one server info result")
+	}
+
+	for _, serverInfo := range results {
+		if len(serverInfo.Addr) == 0 {
+			t.Error("Expected server address to be non empty")
+		}
+		if serverInfo.Error != "" {
+			t.Errorf("Unexpected error = %v\n", serverInfo.Error)
+		}
+		if serverInfo.Data.StorageInfo.Free == 0 {
+			t.Error("Expected StorageInfo.Free to be non empty")
+		}
+		if serverInfo.Data.Properties.Region != globalMinioDefaultRegion {
+			t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, serverInfo.Data.Properties.Region)
+		}
+	}
+}
+
 // TestToAdminAPIErr - test for toAdminAPIErr helper function.
 func TestToAdminAPIErr(t *testing.T) {
 	testCases := []struct {
@@ -1219,6 +1359,11 @@ func TestToAdminAPIErr(t *testing.T) {
 }
 
 func TestWriteSetConfigResponse(t *testing.T) {
+	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeAll(rootPath)
 	testCases := []struct {
 		status bool
 		errs   []error
@@ -1250,7 +1395,7 @@ func TestWriteSetConfigResponse(t *testing.T) {
 		},
 	}
 
-	testURL, err := url.Parse("dummy.com")
+	testURL, err := url.Parse("http://dummy.com")
 	if err != nil {
 		t.Fatalf("Failed to parse a place-holder url")
 	}
@@ -1288,9 +1433,9 @@ func TestWriteSetConfigResponse(t *testing.T) {
 	}
 }
 
-// mkUploadsHealQuery - helper function to construct query values for
+// mkListUploadsHealQuery - helper function to construct query values for
 // listUploadsHeal.
-func mkUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values {
+func mkListUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values {
 
 	queryVal := make(url.Values)
 	queryVal.Set("heal", "")
@@ -1310,7 +1455,7 @@ func TestListHealUploadsHandler(t *testing.T) {
 	}
 	defer adminTestBed.TearDown()
 
-	err = adminTestBed.objLayer.MakeBucket("mybucket")
+	err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}
@@ -1319,12 +1464,13 @@ func TestListHealUploadsHandler(t *testing.T) {
 	defer adminTestBed.objLayer.DeleteBucket("mybucket")
 
 	testCases := []struct {
-		bucket     string
-		prefix     string
-		keyMarker  string
-		delimiter  string
-		maxKeys    string
-		statusCode int
+		bucket       string
+		prefix       string
+		keyMarker    string
+		delimiter    string
+		maxKeys      string
+		statusCode   int
+		expectedResp ListMultipartUploadsResponse
 	}{
 		// 1. Valid params.
 		{
@@ -1334,6 +1480,14 @@ func TestListHealUploadsHandler(t *testing.T) {
 			delimiter:  "/",
 			maxKeys:    "10",
 			statusCode: http.StatusOK,
+			expectedResp: ListMultipartUploadsResponse{
+				XMLName:    xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListMultipartUploadsResult"},
+				Bucket:     "mybucket",
+				KeyMarker:  "prefix11",
+				Delimiter:  "/",
+				Prefix:     "prefix",
+				MaxUploads: 10,
+			},
 		},
 		// 2. Valid params with empty prefix.
 		{
@@ -1343,6 +1497,14 @@ func TestListHealUploadsHandler(t *testing.T) {
 			delimiter:  "/",
 			maxKeys:    "10",
 			statusCode: http.StatusOK,
+			expectedResp: ListMultipartUploadsResponse{
+				XMLName:    xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListMultipartUploadsResult"},
+				Bucket:     "mybucket",
+				KeyMarker:  "",
+				Delimiter:  "/",
+				Prefix:     "",
+				MaxUploads: 10,
+			},
 		},
 		// 3. Invalid params with invalid bucket.
 		{
@@ -1401,23 +1563,64 @@ func TestListHealUploadsHandler(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		queryVal := mkUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys)
+		queryVal := mkListUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys)
 
-		req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil)
+		req, err := buildAdminRequest(queryVal, "list-uploads", http.MethodGet, 0, nil)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to construct list uploads needing heal request - %v", i+1, err)
 		}
-		req.Header.Set(minioAdminOpHeader, "list-uploads")
 
-		cred := serverConfig.GetCredential()
-		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to sign list uploads needing heal request - %v", i+1, err)
-		}
 		rec := httptest.NewRecorder()
 		adminTestBed.mux.ServeHTTP(rec, req)
+
 		if test.statusCode != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
 		}
+
+		// Compare result with the expected one only when we receive 200 OK
+		if rec.Code == http.StatusOK {
+			resp := rec.Result()
+			xmlBytes, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Errorf("Test %d: Failed to read response %v", i+1, err)
+			}
+
+			var actualResult ListMultipartUploadsResponse
+			err = xml.Unmarshal(xmlBytes, &actualResult)
+			if err != nil {
+				t.Errorf("Test %d: Failed to unmarshal xml %v", i+1, err)
+			}
+
+			if !reflect.DeepEqual(test.expectedResp, actualResult) {
+				t.Fatalf("Test %d: Unexpected response `%+v`, expected: `%+v`", i+1, test.expectedResp, actualResult)
+			}
+		}
+
+	}
+}
+
+// Test for newHealResult helper function.
+func TestNewHealResult(t *testing.T) {
+	testCases := []struct {
+		healedDisks  int
+		offlineDisks int
+		state        healState
+	}{
+		// 1. No disks healed, no disks offline.
+		{0, 0, healNone},
+		// 2. No disks healed, non-zero disks offline.
+		{0, 1, healNone},
+		// 3. Non-zero disks healed, no disks offline.
+		{1, 0, healOK},
+		// 4. Non-zero disks healed, non-zero disks offline.
+		{1, 1, healPartial},
+	}
+
+	for i, test := range testCases {
+		actual := newHealResult(test.healedDisks, test.offlineDisks)
+		if actual.State != test.state {
+			t.Errorf("Test %d: Expected %v but received %v", i+1,
+				test.state, actual.State)
+		}
 	}
 }

cmd/admin-router.go

@@ -64,6 +64,8 @@ func registerAdminRouter(mux *router.Router) {
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler)
 	// Heal Format.
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler)
+	// Heal Uploads.
+	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "upload").HandlerFunc(adminAPI.HealUploadHandler)
 
 	/// Config operations
 

cmd/admin-rpc-client.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2014, 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net/url"
+	"net"
 	"os"
 	"path"
 	"path/filepath"
@@ -28,6 +28,8 @@ import (
 	"sort"
 	"sync"
 	"time"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
 const (
@@ -35,7 +37,7 @@ const (
 	serviceRestartRPC = "Admin.Restart"
 	listLocksRPC      = "Admin.ListLocks"
 	reInitDisksRPC    = "Admin.ReInitDisks"
-	uptimeRPC         = "Admin.Uptime"
+	serverInfoDataRPC = "Admin.ServerInfoData"
 	getConfigRPC      = "Admin.GetConfig"
 	writeTmpConfigRPC = "Admin.WriteTmpConfig"
 	commitConfigRPC   = "Admin.CommitConfig"
@@ -57,7 +59,7 @@ type adminCmdRunner interface {
 	Restart() error
 	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
 	ReInitDisks() error
-	Uptime() (time.Duration, error)
+	ServerInfoData() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
 	WriteTmpConfig(tmpFileName string, configBytes []byte) error
 	CommitConfig(tmpFileName string) error
@@ -110,26 +112,48 @@ func (rc remoteAdminClient) ReInitDisks() error {
 	return rc.Call(reInitDisksRPC, &args, &reply)
 }
 
-// Uptime - Returns the uptime of this server. Timestamp is taken
-// after object layer is initialized.
-func (lc localAdminClient) Uptime() (time.Duration, error) {
+// ServerInfoData - Returns the server info of this server.
+func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
 	if globalBootTime.IsZero() {
-		return time.Duration(0), errServerNotInitialized
+		return sid, errServerNotInitialized
 	}
 
-	return time.Now().UTC().Sub(globalBootTime), nil
+	// Build storage info
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		return sid, errServerNotInitialized
+	}
+	storage := objLayer.StorageInfo()
+
+	var arns []string
+	for queueArn := range globalEventNotifier.GetAllExternalTargets() {
+		arns = append(arns, queueArn)
+	}
+
+	return ServerInfoData{
+		StorageInfo: storage,
+		ConnStats:   globalConnStats.toServerConnStats(),
+		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+		Properties: ServerProperties{
+			Uptime:   UTCNow().Sub(globalBootTime),
+			Version:  Version,
+			CommitID: CommitID,
+			SQSARN:   arns,
+			Region:   serverConfig.GetRegion(),
+		},
+	}, nil
 }
 
-// Uptime - returns the uptime of the server to which the RPC call is made.
-func (rc remoteAdminClient) Uptime() (time.Duration, error) {
+// ServerInfo - returns the server info of the server to which the RPC call is made.
+func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
 	args := AuthRPCArgs{}
-	reply := UptimeReply{}
-	err := rc.Call(uptimeRPC, &args, &reply)
+	reply := ServerInfoDataReply{}
+	err := rc.Call(serverInfoDataRPC, &args, &reply)
 	if err != nil {
-		return time.Duration(0), err
+		return sid, err
 	}
 
-	return reply.Uptime, nil
+	return reply.ServerInfoData, nil
 }
 
 // GetConfig - returns config.json of the local server.
@@ -211,52 +235,43 @@ type adminPeer struct {
 type adminPeers []adminPeer
 
 // makeAdminPeers - helper function to construct a collection of adminPeer.
-func makeAdminPeers(eps []*url.URL) adminPeers {
-	var servicePeers []adminPeer
-
-	// map to store peers that are already added to ret
-	seenAddr := make(map[string]bool)
-
-	// add local (self) as peer in the array
-	servicePeers = append(servicePeers, adminPeer{
-		globalMinioAddr,
+func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
+	thisPeer := globalMinioAddr
+	if globalMinioHost == "" {
+		thisPeer = net.JoinHostPort("localhost", globalMinioPort)
+	}
+	adminPeerList = append(adminPeerList, adminPeer{
+		thisPeer,
 		localAdminClient{},
 	})
-	seenAddr[globalMinioAddr] = true
 
-	serverCred := serverConfig.GetCredential()
-	// iterate over endpoints to find new remote peers and add
-	// them to ret.
-	for _, ep := range eps {
-		if ep.Host == "" {
+	hostSet := set.CreateStringSet(globalMinioAddr)
+	cred := serverConfig.GetCredential()
+	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
+	for _, host := range GetRemotePeers(endpoints) {
+		if hostSet.Contains(host) {
 			continue
 		}
-
-		// Check if the remote host has been added already
-		if !seenAddr[ep.Host] {
-			cfg := authConfig{
-				accessKey:       serverCred.AccessKey,
-				secretKey:       serverCred.SecretKey,
-				serverAddr:      ep.Host,
+		hostSet.Add(host)
+		adminPeerList = append(adminPeerList, adminPeer{
+			addr: host,
+			cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
+				accessKey:       cred.AccessKey,
+				secretKey:       cred.SecretKey,
+				serverAddr:      host,
+				serviceEndpoint: serviceEndpoint,
 				secureConn:      globalIsSSL,
-				serviceEndpoint: path.Join(minioReservedBucketPath, adminPath),
 				serviceName:     "Admin",
-			}
-
-			servicePeers = append(servicePeers, adminPeer{
-				addr:      ep.Host,
-				cmdRunner: &remoteAdminClient{newAuthRPCClient(cfg)},
-			})
-			seenAddr[ep.Host] = true
-		}
+			})},
+		})
 	}
 
-	return servicePeers
+	return adminPeerList
 }
 
 // Initialize global adminPeer collection.
-func initGlobalAdminPeers(eps []*url.URL) {
-	globalAdminPeers = makeAdminPeers(eps)
+func initGlobalAdminPeers(endpoints EndpointList) {
+	globalAdminPeers = makeAdminPeers(endpoints)
 }
 
 // invokeServiceCmd - Invoke Restart command.
@@ -380,7 +395,7 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
 	// the setup is the uptime of the single minio server
 	// instance.
 	if !globalIsDistXL {
-		return time.Now().UTC().Sub(globalBootTime), nil
+		return UTCNow().Sub(globalBootTime), nil
 	}
 
 	uptimes := make(uptimeSlice, len(peers))
@@ -391,7 +406,8 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
 		wg.Add(1)
 		go func(idx int, peer adminPeer) {
 			defer wg.Done()
-			uptimes[idx].uptime, uptimes[idx].err = peer.cmdRunner.Uptime()
+			serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
+			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
 		}(i, peer)
 	}
 	wg.Wait()
@@ -477,7 +493,7 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
 
 // getValidServerConfig - finds the server config that is present in
 // quorum or more number of servers.
-func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (serverConfigV13, error) {
+func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (scv serverConfigV13, e error) {
 	// majority-based quorum
 	quorum := len(serverConfigs)/2 + 1
 
@@ -550,7 +566,7 @@ func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (server
 
 	// If quorum nodes don't agree.
 	if maxOccurrence < quorum {
-		return serverConfigV13{}, errXLWriteQuorum
+		return scv, errXLWriteQuorum
 	}
 
 	return configJSON, nil

cmd/admin-rpc-client_test.go

@@ -245,7 +245,7 @@ func TestGetValidServerConfig(t *testing.T) {
 
 	// Invalid config - no quorum.
 	serverConfigs = []serverConfigV13{c1, c2, c2, c1}
-	validConfig, err = getValidServerConfig(serverConfigs, noErrs)
+	_, err = getValidServerConfig(serverConfigs, noErrs)
 	if err != errXLWriteQuorum {
 		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
 	}
@@ -253,7 +253,7 @@ func TestGetValidServerConfig(t *testing.T) {
 	// All errors
 	allErrs := []error{errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound}
 	serverConfigs = []serverConfigV13{{}, {}, {}, {}}
-	validConfig, err = getValidServerConfig(serverConfigs, allErrs)
+	_, err = getValidServerConfig(serverConfigs, allErrs)
 	if err != errXLWriteQuorum {
 		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
 	}

cmd/admin-rpc-server.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"net/rpc"
 	"os"
 	"path/filepath"
 	"time"
@@ -53,10 +52,10 @@ type ListLocksReply struct {
 	volLocks []VolumeLockInfo
 }
 
-// UptimeReply - wraps the uptime response over RPC.
-type UptimeReply struct {
+// ServerInfoDataReply - wraps the server info response over RPC.
+type ServerInfoDataReply struct {
 	AuthRPCReply
-	Uptime time.Duration
+	ServerInfoData ServerInfoData
 }
 
 // ConfigReply - wraps the server config response over RPC.
@@ -122,8 +121,8 @@ func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
 	return nil
 }
 
-// Uptime - returns the time when object layer was initialized on this server.
-func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error {
+// ServerInfo - returns the server info when object layer was initialized on this server.
+func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply) error {
 	if err := args.IsAuthenticated(); err != nil {
 		return err
 	}
@@ -132,12 +131,29 @@ func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error {
 		return errServerNotInitialized
 	}
 
-	// N B The uptime is computed assuming that the system time is
-	// monotonic. This is not the case in time pkg in Go, see
-	// https://github.com/golang/go/issues/12914. This is expected
-	// to be fixed by go1.9.
-	*reply = UptimeReply{
-		Uptime: time.Now().UTC().Sub(globalBootTime),
+	// Build storage info
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		return errServerNotInitialized
+	}
+	storageInfo := objLayer.StorageInfo()
+
+	var arns []string
+	for queueArn := range globalEventNotifier.GetAllExternalTargets() {
+		arns = append(arns, queueArn)
+	}
+
+	reply.ServerInfoData = ServerInfoData{
+		Properties: ServerProperties{
+			Uptime:   UTCNow().Sub(globalBootTime),
+			Version:  Version,
+			CommitID: CommitID,
+			Region:   serverConfig.GetRegion(),
+			SQSARN:   arns,
+		},
+		StorageInfo: storageInfo,
+		ConnStats:   globalConnStats.toServerConnStats(),
+		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
 	}
 
 	return nil
@@ -219,7 +235,7 @@ func (s *adminCmd) CommitConfig(cArgs *CommitConfigArgs, cReply *CommitConfigRep
 // stop and restart commands.
 func registerAdminRPCRouter(mux *router.Router) error {
 	adminRPCHandler := &adminCmd{}
-	adminRPCServer := rpc.NewServer()
+	adminRPCServer := newRPCServer()
 	err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
 	if err != nil {
 		return traceError(err)

cmd/admin-rpc-server_test.go

@@ -18,9 +18,7 @@ package cmd
 
 import (
 	"encoding/json"
-	"net/url"
 	"testing"
-	"time"
 )
 
 func testAdminCmd(cmd cmdType, t *testing.T) {
@@ -40,7 +38,7 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
 		Username:    creds.AccessKey,
 		Password:    creds.SecretKey,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	reply := LoginRPCReply{}
 	err = adminServer.Login(&args, &reply)
@@ -87,9 +85,7 @@ func TestReInitDisks(t *testing.T) {
 	defer removeRoots(xlDirs)
 
 	// Set globalEndpoints for a single node XL setup.
-	for _, xlDir := range xlDirs {
-		globalEndpoints = append(globalEndpoints, &url.URL{Path: xlDir})
-	}
+	globalEndpoints = mustGetNewEndpointList(xlDirs...)
 
 	// Setup admin rpc server for an XL backend.
 	globalIsXL = true
@@ -99,7 +95,7 @@ func TestReInitDisks(t *testing.T) {
 		Username:    creds.AccessKey,
 		Password:    creds.SecretKey,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	reply := LoginRPCReply{}
 	err = adminServer.Login(&args, &reply)
@@ -124,7 +120,7 @@ func TestReInitDisks(t *testing.T) {
 		Username:    creds.AccessKey,
 		Password:    creds.SecretKey,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	fsReply := LoginRPCReply{}
 	err = fsAdminServer.Login(&fsArgs, &fsReply)
@@ -161,7 +157,7 @@ func TestGetConfig(t *testing.T) {
 		Username:    creds.AccessKey,
 		Password:    creds.SecretKey,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	reply := LoginRPCReply{}
 	err = adminServer.Login(&args, &reply)
@@ -205,7 +201,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
 		Username:    creds.AccessKey,
 		Password:    creds.SecretKey,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	reply := LoginRPCReply{}
 	err = adminServer.Login(&args, &reply)

cmd/api-errors.go

@@ -114,6 +114,8 @@ const (
 	ErrInvalidQueryParams
 	ErrBucketAlreadyOwnedByYou
 	ErrInvalidDuration
+	ErrNotSupported
+	ErrBucketAlreadyExists
 	// Add new error codes here.
 
 	// Bucket notification related errors.
@@ -139,6 +141,7 @@ const (
 	ErrObjectExistsAsDirectory
 	ErrPolicyNesting
 	ErrInvalidObjectName
+	ErrInvalidResourceName
 	ErrServerNotInitialized
 	// Add new extended error codes here.
 	// Please open a https://github.com/minio/minio/issues before adding
@@ -147,6 +150,7 @@ const (
 	ErrAdminInvalidAccessKey
 	ErrAdminInvalidSecretKey
 	ErrAdminConfigNoQuorum
+	ErrInsecureClientRequest
 )
 
 // error code to APIError structure, these fields carry respective
@@ -228,7 +232,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
 	ErrInvalidAccessKeyID: {
-		Code:           "InvalidAccessKeyID",
+		Code:           "InvalidAccessKeyId",
 		Description:    "The access key ID you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
@@ -314,7 +318,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInvalidPart: {
 		Code:           "InvalidPart",
-		Description:    "One or more of the specified parts could not be found.",
+		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartOrder: {
@@ -352,6 +356,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The bucket you tried to delete is not empty",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrBucketAlreadyExists: {
+		Code:           "BucketAlreadyExists",
+		Description:    "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again.",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrAllAccessDisabled: {
 		Code:           "AllAccessDisabled",
 		Description:    "All access to this bucket has been disabled.",
@@ -588,7 +597,12 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInvalidObjectName: {
 		Code:           "XMinioInvalidObjectName",
-		Description:    "Object name contains unsupported characters. Unsupported characters are `^*|\\\"",
+		Description:    "Object name contains unsupported characters.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidResourceName: {
+		Code:           "XMinioInvalidResourceName",
+		Description:    "Resource name contains bad components such as \"..\" or \".\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrServerNotInitialized: {
@@ -611,6 +625,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Configuration update failed because server quorum was not met",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrInsecureClientRequest: {
+		Code:           "XMinioInsecureClientRequest",
+		Description:    "Cannot respond to plain-text request from TLS-encrypted server",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// Add your error structure here.
 }
@@ -650,6 +669,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrStorageFull
 	case BadDigest:
 		apiErr = ErrBadDigest
+	case AllAccessDisabled:
+		apiErr = ErrAllAccessDisabled
 	case IncompleteBody:
 		apiErr = ErrIncompleteBody
 	case ObjectExistsAsDirectory:
@@ -660,8 +681,12 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrInvalidBucketName
 	case BucketNotFound:
 		apiErr = ErrNoSuchBucket
+	case BucketAlreadyOwnedByYou:
+		apiErr = ErrBucketAlreadyOwnedByYou
 	case BucketNotEmpty:
 		apiErr = ErrBucketNotEmpty
+	case BucketAlreadyExists:
+		apiErr = ErrBucketAlreadyExists
 	case BucketExists:
 		apiErr = ErrBucketAlreadyOwnedByYou
 	case ObjectNotFound:
@@ -686,14 +711,22 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrNoSuchUpload
 	case PartTooSmall:
 		apiErr = ErrEntityTooSmall
+	case SignatureDoesNotMatch:
+		apiErr = ErrSignatureDoesNotMatch
 	case SHA256Mismatch:
 		apiErr = ErrContentSHA256Mismatch
 	case ObjectTooLarge:
 		apiErr = ErrEntityTooLarge
 	case ObjectTooSmall:
 		apiErr = ErrEntityTooSmall
+	case NotSupported:
+		apiErr = ErrNotSupported
 	case NotImplemented:
 		apiErr = ErrNotImplemented
+	case PolicyNotFound:
+		apiErr = ErrNoSuchBucketPolicy
+	case PartTooBig:
+		apiErr = ErrEntityTooLarge
 	default:
 		apiErr = ErrInternalError
 	}

cmd/api-errors_test.go

@@ -103,6 +103,10 @@ func TestAPIErrCode(t *testing.T) {
 			StorageFull{},
 			ErrStorageFull,
 		},
+		{
+			NotSupported{},
+			ErrNotSupported,
+		},
 		{
 			NotImplemented{},
 			ErrNotImplemented,

cmd/api-headers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015,2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -34,8 +34,13 @@ func mustGetRequestID(t time.Time) string {
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
 	// Set unique request ID for each reply.
-	w.Header().Set(responseRequestIDKey, mustGetRequestID(time.Now().UTC()))
+	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
 	w.Header().Set("Server", globalServerUserAgent)
+	// Set `x-amz-bucket-region` only if region is set on the server
+	// by default minio uses an empty region.
+	if region := serverConfig.GetRegion(); region != "" {
+		w.Header().Set("X-Amz-Bucket-Region", region)
+	}
 	w.Header().Set("Accept-Ranges", "bytes")
 }
 
@@ -61,8 +66,8 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 	w.Header().Set("Last-Modified", lastModified)
 
 	// Set Etag if available.
-	if objInfo.MD5Sum != "" {
-		w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+	if objInfo.ETag != "" {
+		w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
 	}
 
 	// Set all other user defined metadata.

cmd/api-headers_test.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,12 +18,11 @@ package cmd
 
 import (
 	"testing"
-	"time"
 )
 
 func TestNewRequestID(t *testing.T) {
 	// Ensure that it returns an alphanumeric result of length 16.
-	var id = mustGetRequestID(time.Now().UTC())
+	var id = mustGetRequestID(UTCNow())
 
 	if len(id) != 16 {
 		t.Fail()

cmd/api-response.go

@@ -288,8 +288,6 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	var owner = Owner{}
 
 	owner.ID = globalMinioDefaultOwnerID
-	owner.DisplayName = globalMinioDefaultOwnerID
-
 	for _, bucket := range buckets {
 		var listbucket = Bucket{}
 		listbucket.Name = bucket.Name
@@ -312,8 +310,6 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 	var data = ListObjectsResponse{}
 
 	owner.ID = globalMinioDefaultOwnerID
-	owner.DisplayName = globalMinioDefaultOwnerID
-
 	for _, object := range resp.Objects {
 		var content = Object{}
 		if object.Name == "" {
@@ -321,8 +317,8 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 		}
 		content.Key = object.Name
 		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
-		if object.MD5Sum != "" {
-			content.ETag = "\"" + object.MD5Sum + "\""
+		if object.ETag != "" {
+			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
 		content.StorageClass = globalMinioDefaultStorageClass
@@ -352,26 +348,25 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, fetchOwner bool, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
 	var contents []Object
-	var prefixes []CommonPrefix
+	var commonPrefixes []CommonPrefix
 	var owner = Owner{}
 	var data = ListObjectsV2Response{}
 
 	if fetchOwner {
 		owner.ID = globalMinioDefaultOwnerID
-		owner.DisplayName = globalMinioDefaultOwnerID
 	}
 
-	for _, object := range resp.Objects {
+	for _, object := range objects {
 		var content = Object{}
 		if object.Name == "" {
 			continue
 		}
 		content.Key = object.Name
 		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
-		if object.MD5Sum != "" {
-			content.ETag = "\"" + object.MD5Sum + "\""
+		if object.ETag != "" {
+			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
 		content.StorageClass = globalMinioDefaultStorageClass
@@ -387,14 +382,14 @@ func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter
 	data.Prefix = prefix
 	data.MaxKeys = maxKeys
 	data.ContinuationToken = token
-	data.NextContinuationToken = resp.NextMarker
-	data.IsTruncated = resp.IsTruncated
-	for _, prefix := range resp.Prefixes {
+	data.NextContinuationToken = nextToken
+	data.IsTruncated = isTruncated
+	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = prefix
-		prefixes = append(prefixes, prefixItem)
+		commonPrefixes = append(commonPrefixes, prefixItem)
 	}
-	data.CommonPrefixes = prefixes
+	data.CommonPrefixes = commonPrefixes
 	data.KeyCount = len(data.Contents) + len(data.CommonPrefixes)
 	return data
 }
@@ -443,9 +438,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
 	listPartsResponse.UploadID = partsInfo.UploadID
 	listPartsResponse.StorageClass = globalMinioDefaultStorageClass
 	listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID
-	listPartsResponse.Initiator.DisplayName = globalMinioDefaultOwnerID
 	listPartsResponse.Owner.ID = globalMinioDefaultOwnerID
-	listPartsResponse.Owner.DisplayName = globalMinioDefaultOwnerID
 
 	listPartsResponse.MaxParts = partsInfo.MaxParts
 	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker

cmd/auth-handler.go

@@ -64,7 +64,6 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool {
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
 func isRequestSignStreamingV4(r *http.Request) bool {
 	return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
-		r.Header.Get("content-encoding") == streamingContentEncoding &&
 		r.Method == httpPUT
 }
 
@@ -114,13 +113,13 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string)
 		// Signature V2 validation.
 		s3Error := isReqAuthenticatedV2(r)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 		}
 		return s3Error
 	case authTypeSigned, authTypePresigned:
 		s3Error := isReqAuthenticated(r, region)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 		}
 		return s3Error
 	}
@@ -143,19 +142,16 @@ func isReqAuthenticatedV2(r *http.Request) (s3Error APIErrorCode) {
 	return doesPresignV2SignatureMatch(r)
 }
 
-func reqSignatureV4Verify(r *http.Request) (s3Error APIErrorCode) {
-	sha256sum := r.Header.Get("X-Amz-Content-Sha256")
-	// Skips calculating sha256 on the payload on server,
-	// if client requested for it.
-	if skipContentSha256Cksum(r) {
-		sha256sum = unsignedPayload
+func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode) {
+	sha256sum := getContentSha256Cksum(r)
+	switch {
+	case isRequestSignatureV4(r):
+		return doesSignatureMatch(sha256sum, r, region)
+	case isRequestPresignedSignatureV4(r):
+		return doesPresignedSignatureMatch(sha256sum, r, region)
+	default:
+		return ErrAccessDenied
 	}
-	if isRequestSignatureV4(r) {
-		return doesSignatureMatch(sha256sum, r, serverConfig.GetRegion())
-	} else if isRequestPresignedSignatureV4(r) {
-		return doesPresignedSignatureMatch(sha256sum, r, serverConfig.GetRegion())
-	}
-	return ErrAccessDenied
 }
 
 // Verify if request has valid AWS Signature Version '4'.
@@ -163,32 +159,39 @@ func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
 	if r == nil {
 		return ErrInternalError
 	}
+	if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
+		return errCode
+	}
 	payload, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		errorIf(err, "Unable to read request body for signature verification")
 		return ErrInternalError
 	}
+
+	// Populate back the payload.
+	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+
 	// Verify Content-Md5, if payload is set.
 	if r.Header.Get("Content-Md5") != "" {
 		if r.Header.Get("Content-Md5") != getMD5HashBase64(payload) {
 			return ErrBadDigest
 		}
 	}
-	// Populate back the payload.
-	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
-	// Skips calculating sha256 on the payload on server, if client requested for it.
-	var sha256sum string
+
 	if skipContentSha256Cksum(r) {
-		sha256sum = unsignedPayload
-	} else {
-		sha256sum = getSHA256Hash(payload)
+		return ErrNone
 	}
-	if isRequestSignatureV4(r) {
-		return doesSignatureMatch(sha256sum, r, region)
-	} else if isRequestPresignedSignatureV4(r) {
-		return doesPresignedSignatureMatch(sha256sum, r, region)
+
+	// Verify that X-Amz-Content-Sha256 Header == sha256(payload)
+	// If X-Amz-Content-Sha256 header is not sent then we don't calculate/verify sha256(payload)
+	sum := r.Header.Get("X-Amz-Content-Sha256")
+	if isRequestPresignedSignatureV4(r) {
+		sum = r.URL.Query().Get("X-Amz-Content-Sha256")
 	}
-	return ErrAccessDenied
+	if sum != "" && sum != getSHA256Hash(payload) {
+		return ErrContentSHA256Mismatch
+	}
+	return ErrNone
 }
 
 // authHandler - handles all the incoming authorization headers and validates them if possible.

cmd/auth-handler_test.go

@@ -308,6 +308,16 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod
 	return req
 }
 
+func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
+	req := mustNewRequest(method, urlStr, contentLength, body, t)
+	req.Header.Set("Content-Md5", "YWFhYWFhYWFhYWFhYWFhCg==")
+	cred := serverConfig.GetCredential()
+	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
+		t.Fatalf("Unable to initialized new signed http request %s", err)
+	}
+	return req
+}
+
 // Tests is requested authenticated function, tests replies for s3 errors.
 func TestIsReqAuthenticated(t *testing.T) {
 	path, err := newTestConfig(globalMinioDefaultRegion)
@@ -333,16 +343,13 @@ func TestIsReqAuthenticated(t *testing.T) {
 		// When request is unsigned, access denied is returned.
 		{mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrAccessDenied},
 		// When request is properly signed, but has bad Content-MD5 header.
-		{mustNewSignedRequest("PUT", "http://127.0.0.1:9000", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest},
+		{mustNewSignedBadMD5Request("PUT", "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest},
 		// When request is properly signed, error is none.
 		{mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrNone},
 	}
 
 	// Validates all testcases.
 	for _, testCase := range testCases {
-		if testCase.s3Error == ErrBadDigest {
-			testCase.req.Header.Set("Content-Md5", "garbage")
-		}
 		if s3Error := isReqAuthenticated(testCase.req, serverConfig.GetRegion()); s3Error != testCase.s3Error {
 			t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
 		}

cmd/auth-rpc-client.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -52,10 +52,10 @@ type authConfig struct {
 
 // AuthRPCClient is a authenticated RPC client which does authentication before doing Call().
 type AuthRPCClient struct {
-	sync.Mutex            // Mutex to lock this object.
-	rpcClient  *RPCClient // Reconnectable RPC client to make any RPC call.
-	config     authConfig // Authentication configuration information.
-	authToken  string     // Authentication token.
+	sync.RWMutex            // Mutex to lock this object.
+	rpcClient    *RPCClient // Reconnectable RPC client to make any RPC call.
+	config       authConfig // Authentication configuration information.
+	authToken    string     // Authentication token.
 }
 
 // newAuthRPCClient - returns a JWT based authenticated (go) rpc client, which does automatic reconnect.
@@ -78,33 +78,43 @@ func newAuthRPCClient(config authConfig) *AuthRPCClient {
 	}
 }
 
-// Login - a jwt based authentication is performed with rpc server.
+// Login a JWT based authentication is performed with rpc server.
 func (authClient *AuthRPCClient) Login() (err error) {
+	// Login should be attempted one at a time.
+	//
+	// The reason for large region lock here is
+	// to avoid two simultaneous login attempts
+	// racing over each other.
+	//
+	// #1 Login() gets the lock proceeds to login.
+	// #2 Login() waits for the unlock to happen
+	//    after login in #1.
+	// #1 Successfully completes login saves the
+	//    newly acquired token.
+	// #2 Successfully gets the lock and proceeds,
+	//    but since we have acquired the token
+	//    already the call quickly returns.
 	authClient.Lock()
 	defer authClient.Unlock()
 
-	// Return if already logged in.
-	if authClient.authToken != "" {
-		return nil
+	// Attempt to login if not logged in already.
+	if authClient.authToken == "" {
+		// Login to authenticate and acquire a new auth token.
+		var (
+			loginMethod = authClient.config.serviceName + loginMethodName
+			loginArgs   = LoginRPCArgs{
+				Username:    authClient.config.accessKey,
+				Password:    authClient.config.secretKey,
+				Version:     Version,
+				RequestTime: UTCNow(),
+			}
+			loginReply = LoginRPCReply{}
+		)
+		if err = authClient.rpcClient.Call(loginMethod, &loginArgs, &loginReply); err != nil {
+			return err
+		}
+		authClient.authToken = loginReply.AuthToken
 	}
-
-	// Call login.
-	args := LoginRPCArgs{
-		Username:    authClient.config.accessKey,
-		Password:    authClient.config.secretKey,
-		Version:     Version,
-		RequestTime: time.Now().UTC(),
-	}
-
-	reply := LoginRPCReply{}
-	serviceMethod := authClient.config.serviceName + loginMethodName
-	if err = authClient.rpcClient.Call(serviceMethod, &args, &reply); err != nil {
-		return err
-	}
-
-	// Logged in successfully.
-	authClient.authToken = reply.AuthToken
-
 	return nil
 }
 
@@ -112,17 +122,17 @@ func (authClient *AuthRPCClient) Login() (err error) {
 func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
 	SetAuthToken(authToken string)
 }, reply interface{}) (err error) {
-	// On successful login, execute RPC call.
-	if err = authClient.Login(); err == nil {
-		authClient.Lock()
-		// Set token and timestamp before the rpc call.
-		args.SetAuthToken(authClient.authToken)
-		authClient.Unlock()
+	if err = authClient.Login(); err != nil {
+		return err
+	} // On successful login, execute RPC call.
 
-		// Do RPC call.
-		err = authClient.rpcClient.Call(serviceMethod, args, reply)
-	}
-	return err
+	authClient.RLock()
+	// Set token before the rpc call.
+	args.SetAuthToken(authClient.authToken)
+	authClient.RUnlock()
+
+	// Do an RPC call.
+	return authClient.rpcClient.Call(serviceMethod, args, reply)
 }
 
 // Call executes RPC call till success or globalAuthRPCRetryThreshold on ErrShutdown.

cmd/auth-rpc-server_test.go

@@ -77,7 +77,7 @@ func TestLogin(t *testing.T) {
 		// Invalid password length
 		{
 			args: LoginRPCArgs{
-				Username: globalMinioDefaultOwnerID,
+				Username: "minio",
 				Password: "aaa",
 				Version:  Version,
 			},

cmd/benchmark-utils_test.go

@@ -23,7 +23,6 @@ import (
 	"math/rand"
 	"strconv"
 	"testing"
-	"time"
 
 	humanize "github.com/dustin/go-humanize"
 )
@@ -40,7 +39,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -50,7 +49,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	metadata := make(map[string]string)
-	metadata["md5Sum"] = getMD5Hash(textData)
+	metadata["etag"] = getMD5Hash(textData)
 	sha256sum := ""
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
@@ -62,8 +61,8 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.MD5Sum != metadata["md5Sum"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		if objInfo.ETag != metadata["etag"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
 		}
 	}
 	// Benchmark ends here. Stop timer.
@@ -79,22 +78,22 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
 
 	objSize := 128 * humanize.MiByte
 
-	// PutObjectPart returns md5Sum of the object inserted.
-	// md5Sum variable is assigned with that value.
-	var md5Sum, uploadID string
+	// PutObjectPart returns etag of the object inserted.
+	// etag variable is assigned with that value.
+	var etag, uploadID string
 	// get text data generated for number of bytes equal to object size.
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
 	metadata := make(map[string]string)
-	metadata["md5Sum"] = getMD5Hash(textData)
+	metadata["etag"] = getMD5Hash(textData)
 	sha256sum := ""
 	uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
 	if err != nil {
@@ -116,14 +115,14 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 				textPartData = textData[j*partSize:]
 			}
 			metadata := make(map[string]string)
-			metadata["md5Sum"] = getMD5Hash([]byte(textPartData))
+			metadata["etag"] = getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
-			partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"], sha256sum)
+			partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["etag"], sha256sum)
 			if err != nil {
 				b.Fatal(err)
 			}
-			if partInfo.ETag != metadata["md5Sum"] {
-				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+			if partInfo.ETag != metadata["etag"] {
+				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, metadata["etag"])
 			}
 		}
 	}
@@ -200,7 +199,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -209,19 +208,19 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < 10; i++ {
 		// get text data generated for number of bytes equal to object size.
 		textData := generateBytesData(objSize)
-		// generate md5sum for the generated data.
-		// md5sum of the data to written is required as input for PutObject.
+		// generate etag for the generated data.
+		// etag of the data to written is required as input for PutObject.
 		// PutObject is the functions which writes the data onto the FS/XL backend.
 		metadata := make(map[string]string)
-		metadata["md5Sum"] = getMD5Hash(textData)
+		metadata["etag"] = getMD5Hash(textData)
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.MD5Sum != metadata["md5Sum"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		if objInfo.ETag != metadata["etag"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
 		}
 	}
 
@@ -245,7 +244,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 func getRandomByte() []byte {
 	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 	// seeding the random number generator.
-	rand.Seed(time.Now().UTC().UnixNano())
+	rand.Seed(UTCNow().UnixNano())
 	var b byte
 	// pick a character randomly.
 	b = letterBytes[rand.Intn(len(letterBytes))]
@@ -308,7 +307,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -318,7 +317,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	metadata := make(map[string]string)
-	metadata["md5Sum"] = getMD5Hash([]byte(textData))
+	metadata["etag"] = getMD5Hash([]byte(textData))
 	sha256sum := ""
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
@@ -333,8 +332,8 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 			if err != nil {
 				b.Fatal(err)
 			}
-			if objInfo.MD5Sum != metadata["md5Sum"] {
-				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.MD5Sum, metadata["md5Sum"])
+			if objInfo.ETag != metadata["etag"] {
+				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, metadata["etag"])
 			}
 			i++
 		}
@@ -356,7 +355,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -368,7 +367,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// md5sum of the data to written is required as input for PutObject.
 		// PutObject is the functions which writes the data onto the FS/XL backend.
 		metadata := make(map[string]string)
-		metadata["md5Sum"] = getMD5Hash([]byte(textData))
+		metadata["etag"] = getMD5Hash([]byte(textData))
 		sha256sum := ""
 		// insert the object.
 		var objInfo ObjectInfo
@@ -376,8 +375,8 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.MD5Sum != metadata["md5Sum"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		if objInfo.ETag != metadata["etag"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
 		}
 	}
 

cmd/browser-flag.go

@@ -0,0 +1,68 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// BrowserFlag - wrapper bool type.
+type BrowserFlag bool
+
+// String - returns string of BrowserFlag.
+func (bf BrowserFlag) String() string {
+	if bf {
+		return "on"
+	}
+
+	return "off"
+}
+
+// MarshalJSON - converts BrowserFlag into JSON data.
+func (bf BrowserFlag) MarshalJSON() ([]byte, error) {
+	return json.Marshal(bf.String())
+}
+
+// UnmarshalJSON - parses given data into BrowserFlag.
+func (bf *BrowserFlag) UnmarshalJSON(data []byte) (err error) {
+	var s string
+	if err = json.Unmarshal(data, &s); err == nil {
+		b := BrowserFlag(true)
+		if s == "" {
+			// Empty string is treated as valid.
+			*bf = b
+		} else if b, err = ParseBrowserFlag(s); err == nil {
+			*bf = b
+		}
+	}
+
+	return err
+}
+
+// ParseBrowserFlag - parses string into BrowserFlag.
+func ParseBrowserFlag(s string) (bf BrowserFlag, err error) {
+	if s == "on" {
+		bf = true
+	} else if s == "off" {
+		bf = false
+	} else {
+		err = fmt.Errorf("invalid value ‘%s’ for BrowserFlag", s)
+	}
+
+	return bf, err
+}

cmd/browser-flag_test.go

@@ -0,0 +1,137 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"testing"
+)
+
+// Test BrowserFlag.String()
+func TestBrowserFlagString(t *testing.T) {
+	var bf BrowserFlag
+
+	testCases := []struct {
+		flag           BrowserFlag
+		expectedResult string
+	}{
+		{bf, "off"},
+		{BrowserFlag(true), "on"},
+		{BrowserFlag(false), "off"},
+	}
+
+	for _, testCase := range testCases {
+		str := testCase.flag.String()
+		if testCase.expectedResult != str {
+			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, str)
+		}
+	}
+}
+
+// Test BrowserFlag.MarshalJSON()
+func TestBrowserFlagMarshalJSON(t *testing.T) {
+	var bf BrowserFlag
+
+	testCases := []struct {
+		flag           BrowserFlag
+		expectedResult string
+	}{
+		{bf, `"off"`},
+		{BrowserFlag(true), `"on"`},
+		{BrowserFlag(false), `"off"`},
+	}
+
+	for _, testCase := range testCases {
+		data, _ := testCase.flag.MarshalJSON()
+		if testCase.expectedResult != string(data) {
+			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, string(data))
+		}
+	}
+}
+
+// Test BrowserFlag.UnmarshalJSON()
+func TestBrowserFlagUnmarshalJSON(t *testing.T) {
+	testCases := []struct {
+		data           []byte
+		expectedResult BrowserFlag
+		expectedErr    error
+	}{
+		{[]byte(`{}`), BrowserFlag(false), errors.New("json: cannot unmarshal object into Go value of type string")},
+		{[]byte(`["on"]`), BrowserFlag(false), errors.New("json: cannot unmarshal array into Go value of type string")},
+		{[]byte(`"junk"`), BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
+		{[]byte(`"true"`), BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
+		{[]byte(`"false"`), BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
+		{[]byte(`"ON"`), BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
+		{[]byte(`"OFF"`), BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
+		{[]byte(`""`), BrowserFlag(true), nil},
+		{[]byte(`"on"`), BrowserFlag(true), nil},
+		{[]byte(`"off"`), BrowserFlag(false), nil},
+	}
+
+	for _, testCase := range testCases {
+		var flag BrowserFlag
+		err := (&flag).UnmarshalJSON(testCase.data)
+		if testCase.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+		} else if testCase.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+
+		if err == nil && testCase.expectedResult != flag {
+			t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, flag)
+		}
+	}
+}
+
+// Test ParseBrowserFlag()
+func TestParseBrowserFlag(t *testing.T) {
+	testCases := []struct {
+		flagStr        string
+		expectedResult BrowserFlag
+		expectedErr    error
+	}{
+		{"", BrowserFlag(false), errors.New("invalid value ‘’ for BrowserFlag")},
+		{"junk", BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
+		{"true", BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
+		{"false", BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
+		{"ON", BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
+		{"OFF", BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
+		{"on", BrowserFlag(true), nil},
+		{"off", BrowserFlag(false), nil},
+	}
+
+	for _, testCase := range testCases {
+		bf, err := ParseBrowserFlag(testCase.flagStr)
+		if testCase.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+		} else if testCase.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+
+		if err == nil && testCase.expectedResult != bf {
+			t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, bf)
+		}
+	}
+}

cmd/browser-peer-rpc_test.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,7 +19,6 @@ package cmd
 import (
 	"path"
 	"testing"
-	"time"
 )
 
 // API suite container common to both FS and XL.
@@ -96,7 +95,7 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 		Username:    creds.AccessKey,
 		Password:    creds.SecretKey,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	rreply := &LoginRPCReply{}
 	err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply)
@@ -111,7 +110,7 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 		Username:    creds.AccessKey,
 		Password:    creds.SecretKey,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	rreply = &LoginRPCReply{}
 	err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply)

cmd/browser-rpc-router.go

@@ -17,8 +17,6 @@
 package cmd
 
 import (
-	"net/rpc"
-
 	router "github.com/gorilla/mux"
 )
 
@@ -39,7 +37,7 @@ type browserPeerAPIHandlers struct {
 func registerBrowserPeerRPCRouter(mux *router.Router) error {
 	bpHandlers := &browserPeerAPIHandlers{}
 
-	bpRPCServer := rpc.NewServer()
+	bpRPCServer := newRPCServer()
 	err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)
 	if err != nil {
 		return traceError(err)

cmd/bucket-handlers-listobjects.go

@@ -100,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, fetchOwner, maxKeys, listObjectsInfo)
+	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsInfo.NextMarker, startAfter, delimiter, fetchOwner, listObjectsInfo.IsTruncated, maxKeys, listObjectsInfo.Objects, listObjectsInfo.Prefixes)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))

cmd/bucket-handlers.go

@@ -20,6 +20,7 @@ import (
 	"encoding/base64"
 	"encoding/xml"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"path"
@@ -82,6 +83,10 @@ func enforceBucketPolicy(bucket, action, resource, referer string, queryParams u
 
 // Check if the action is allowed on the bucket/prefix.
 func isBucketActionAllowed(action, bucket, prefix string) bool {
+	if globalBucketPolicies == nil {
+		return false
+	}
+
 	policy := globalBucketPolicies.GetBucketPolicy(bucket)
 	if policy == nil {
 		return false
@@ -322,6 +327,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 
+	// Get host and port from Request.RemoteAddr failing which
+	// fill them with empty strings.
+	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		host, port = "", ""
+	}
+
 	// Notify deleted event for objects.
 	for _, dobj := range deletedObjects {
 		eventNotify(eventData{
@@ -331,6 +343,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 				Name: dobj.ObjectName,
 			},
 			ReqParams: extractReqParams(r),
+			UserAgent: r.UserAgent(),
+			Host:      host,
+			Port:      port,
 		})
 	}
 }
@@ -359,19 +374,26 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	// Validate if incoming location constraint is valid, reject
-	// requests which do not follow valid region requirements.
-	if s3Error := isValidLocationConstraint(r); s3Error != ErrNone {
+	// Parse incoming location constraint.
+	location, s3Error := parseLocationConstraint(r)
+	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
+	// Validate if location sent by the client is valid, reject
+	// requests which do not follow valid region requirements.
+	if !isValidLocation(location) {
+		writeErrorResponse(w, ErrInvalidRegion, r.URL)
+		return
+	}
+
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
 	bucketLock.Lock()
 	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
-	err := objectAPI.MakeBucket(bucket)
+	err := objectAPI.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		errorIf(err, "Unable to create a bucket.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -503,7 +525,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	// Extract metadata to be saved from received Form.
-	metadata := extractMetadataFromForm(formValues)
+	metadata, err := extractMetadataFromHeader(formValues)
+	if err != nil {
+		errorIf(err, "found invalid http request header")
+		writeErrorResponse(w, ErrInternalError, r.URL)
+		return
+	}
 	sha256sum := ""
 
 	objectLock := globalNSMutex.NewNSLock(bucket, object)
@@ -517,15 +544,24 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	w.Header().Set("ETag", `"`+objInfo.MD5Sum+`"`)
+	w.Header().Set("ETag", `"`+objInfo.ETag+`"`)
 	w.Header().Set("Location", getObjectLocation(bucket, object))
 
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		host, port = "", ""
+	}
+
 	// Notify object created event.
 	defer eventNotify(eventData{
 		Type:      ObjectCreatedPost,
 		Bucket:    objInfo.Bucket,
 		ObjInfo:   objInfo,
 		ReqParams: extractReqParams(r),
+		UserAgent: r.UserAgent(),
+		Host:      host,
+		Port:      port,
 	})
 
 	if successRedirect != "" {
@@ -541,7 +577,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		resp := encodeResponse(PostResponse{
 			Bucket:   objInfo.Bucket,
 			Key:      objInfo.Name,
-			ETag:     `"` + objInfo.MD5Sum + `"`,
+			ETag:     `"` + objInfo.ETag + `"`,
 			Location: getObjectLocation(objInfo.Bucket, objInfo.Name),
 		})
 		writeResponse(w, http.StatusCreated, resp, "application/xml")
@@ -617,12 +653,21 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	// Delete bucket access policy, if present - ignore any errors.
 	_ = removeBucketPolicy(bucket, objectAPI)
 
+	// Notify all peers (including self) to update in-memory state
+	S3PeersUpdateBucketPolicy(bucket, policyChange{true, nil})
+
 	// Delete notification config, if present - ignore any errors.
 	_ = removeNotificationConfig(bucket, objectAPI)
 
+	// Notify all peers (including self) to update in-memory state
+	S3PeersUpdateBucketNotification(bucket, nil)
+
 	// Delete listener config, if present - ignore any errors.
 	_ = removeListenerConfig(bucket, objectAPI)
 
+	// Notify all peers (including self) to update in-memory state
+	S3PeersUpdateBucketListener(bucket, []listenerConfig{})
+
 	// Write success response.
 	writeSuccessNoContent(w)
 }

cmd/bucket-handlers_test.go

@@ -68,7 +68,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 			locationResponse:   []byte(""),
 			errorResponse: APIErrorResponse{
 				Resource: "/" + bucketName + "/",
-				Code:     "InvalidAccessKeyID",
+				Code:     "InvalidAccessKeyId",
 				Message:  "The access key ID you provided does not exist in our records.",
 			},
 			shouldPass: false,
@@ -771,3 +771,36 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	// `ExecObjectLayerAPINilTest` manages the operation.
 	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
 }
+
+func TestIsBucketActionAllowed(t *testing.T) {
+	ExecObjectLayerAPITest(t, testIsBucketActionAllowedHandler, []string{"BucketLocation"})
+}
+
+func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+
+	testCases := []struct {
+		// input.
+		action              string
+		bucket              string
+		prefix              string
+		isGlobalPoliciesNil bool
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		{"s3:GetBucketLocation", "mybucket", "abc", true, false},
+		{"s3:ListObject", "mybucket", "abc", false, false},
+	}
+	for i, testCase := range testCases {
+		if testCase.isGlobalPoliciesNil {
+			globalBucketPolicies = nil
+		} else {
+			initBucketPolicies(obj)
+		}
+		isAllowed := isBucketActionAllowed(testCase.action, testCase.bucket, testCase.prefix)
+		if isAllowed != testCase.shouldPass {
+			t.Errorf("Case %d: Expected the response status to be `%t`, but instead found `%t`", i+1, testCase.shouldPass, isAllowed)
+		}
+
+	}
+}

cmd/bucket-notification-datatypes.go

@@ -94,6 +94,10 @@ const (
 	ObjectCreatedCompleteMultipartUpload
 	// ObjectRemovedDelete is s3:ObjectRemoved:Delete
 	ObjectRemovedDelete
+	// ObjectAccessedGet is s3:ObjectAccessed:Get
+	ObjectAccessedGet
+	// ObjectAccessedHead is s3:ObjectAccessed:Head
+	ObjectAccessedHead
 )
 
 // Stringer interface for event name.
@@ -109,6 +113,10 @@ func (eventName EventName) String() string {
 		return "s3:ObjectCreated:CompleteMultipartUpload"
 	case ObjectRemovedDelete:
 		return "s3:ObjectRemoved:Delete"
+	case ObjectAccessedGet:
+		return "s3:ObjectAccessed:Get"
+	case ObjectAccessedHead:
+		return "s3:ObjectAccessed:Head"
 	default:
 		return "s3:Unknown"
 	}
@@ -128,11 +136,13 @@ type bucketMeta struct {
 
 // Notification event object metadata.
 type objectMeta struct {
-	Key       string `json:"key"`
-	Size      int64  `json:"size,omitempty"`
-	ETag      string `json:"eTag,omitempty"`
-	VersionID string `json:"versionId,omitempty"`
-	Sequencer string `json:"sequencer"`
+	Key         string            `json:"key"`
+	Size        int64             `json:"size,omitempty"`
+	ETag        string            `json:"eTag,omitempty"`
+	ContentType string            `json:"contentType,omitempty"`
+	UserDefined map[string]string `json:"userDefined,omitempty"`
+	VersionID   string            `json:"versionId,omitempty"`
+	Sequencer   string            `json:"sequencer"`
 }
 
 const (
@@ -168,6 +178,14 @@ const (
 	eventVersion = "2.0"
 )
 
+// sourceInfo represents information on the client that triggered the
+// event notification.
+type sourceInfo struct {
+	Host      string `json:"host"`
+	Port      string `json:"port"`
+	UserAgent string `json:"userAgent"`
+}
+
 // NotificationEvent represents an Amazon an S3 bucket notification event.
 type NotificationEvent struct {
 	EventVersion      string            `json:"eventVersion"`
@@ -179,6 +197,7 @@ type NotificationEvent struct {
 	RequestParameters map[string]string `json:"requestParameters"`
 	ResponseElements  map[string]string `json:"responseElements"`
 	S3                eventMeta         `json:"s3"`
+	Source            sourceInfo        `json:"source"`
 }
 
 // Represents the minio sqs type and account id's.

cmd/bucket-notification-handlers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -204,6 +204,15 @@ func writeNotification(w http.ResponseWriter, notification map[string][]Notifica
 	if err != nil {
 		return err
 	}
+
+	// https://github.com/containous/traefik/issues/560
+	// https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
+	//
+	// Proxies might buffer the connection to avoid this we
+	// need the proper MIME type before writing to client.
+	// This MIME header tells the proxies to avoid buffering
+	w.Header().Set("Content-Type", "text/event-stream")
+
 	// Add additional CRLF characters for client to
 	// differentiate the individual events properly.
 	_, err = w.Write(append(notificationBytes, crlf...))
@@ -285,7 +294,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	accountID := fmt.Sprintf("%d", time.Now().UTC().UnixNano())
+	accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
 	accountARN := fmt.Sprintf(
 		"%s:%s:%s:%s-%s",
 		minioTopic,

cmd/bucket-notification-handlers_test.go

@@ -185,7 +185,7 @@ func testGetBucketNotificationHandler(obj ObjectLayer, instanceType, bucketName
 	filterRules := []filterRule{
 		{
 			Name:  "prefix",
-			Value: globalMinioDefaultOwnerID,
+			Value: "minio",
 		},
 		{
 			Name:  "suffix",
@@ -268,6 +268,7 @@ func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucke
 			[]string{"*.jpg"}, []string{
 				"s3:ObjectCreated:*",
 				"s3:ObjectRemoved:*",
+				"s3:ObjectAccessed:*",
 			}), 0, nil, credentials.AccessKey, credentials.SecretKey)
 	if tErr != nil {
 		t.Fatalf("%s: Failed to create HTTP testRequest for ListenBucketNotification: <ERROR> %v", instanceType, tErr)

cmd/bucket-notification-utils.go

@@ -16,7 +16,12 @@
 
 package cmd
 
-import "strings"
+import (
+	"errors"
+	"strings"
+
+	"github.com/minio/minio-go/pkg/set"
+)
 
 // List of valid event types.
 var suppportedEventTypes = map[string]struct{}{
@@ -29,6 +34,9 @@ var suppportedEventTypes = map[string]struct{}{
 	// Object removed event types.
 	"s3:ObjectRemoved:*":      {},
 	"s3:ObjectRemoved:Delete": {},
+	"s3:ObjectAccessed:Get":   {},
+	"s3:ObjectAccessed:Head":  {},
+	"s3:ObjectAccessed:*":     {},
 }
 
 // checkEvent - checks if an event is supported.
@@ -104,20 +112,21 @@ func checkARN(arn, arnType string) APIErrorCode {
 	if !strings.HasPrefix(arn, arnType) {
 		return ErrARNNotification
 	}
-	if !strings.HasPrefix(arn, arnType+serverConfig.GetRegion()+":") {
-		return ErrRegionNotification
-	}
-	account := strings.SplitN(strings.TrimPrefix(arn, arnType+serverConfig.GetRegion()+":"), ":", 2)
-	switch len(account) {
-	case 1:
-		// This means ARN is malformed, account should have min of 2elements.
+	strs := strings.SplitN(arn, ":", -1)
+	if len(strs) != 6 {
 		return ErrARNNotification
-	case 2:
-		// Account topic id or topic name cannot be empty.
-		if account[0] == "" || account[1] == "" {
-			return ErrARNNotification
+	}
+	if serverConfig.GetRegion() != "" {
+		region := strs[3]
+		if region != serverConfig.GetRegion() {
+			return ErrRegionNotification
 		}
 	}
+	accountID := strs[4]
+	resource := strs[5]
+	if accountID == "" || resource == "" {
+		return ErrARNNotification
+	}
 	return ErrNone
 }
 
@@ -135,6 +144,9 @@ func isValidQueueID(queueARN string) bool {
 	if isAMQPQueue(sqsARN) { // AMQP eueue.
 		amqpN := serverConfig.Notify.GetAMQPByID(sqsARN.AccountID)
 		return amqpN.Enable && amqpN.URL != ""
+	} else if isMQTTQueue(sqsARN) {
+		mqttN := serverConfig.Notify.GetMQTTByID(sqsARN.AccountID)
+		return mqttN.Enable && mqttN.Broker != ""
 	} else if isNATSQueue(sqsARN) {
 		natsN := serverConfig.Notify.GetNATSByID(sqsARN.AccountID)
 		return natsN.Enable && natsN.Address != ""
@@ -148,6 +160,10 @@ func isValidQueueID(queueARN string) bool {
 		pgN := serverConfig.Notify.GetPostgreSQLByID(sqsARN.AccountID)
 		// Postgres can work with only default conn. info.
 		return pgN.Enable
+	} else if isMySQLQueue(sqsARN) {
+		msqlN := serverConfig.Notify.GetMySQLByID(sqsARN.AccountID)
+		// Mysql can work with only default conn. info.
+		return msqlN.Enable
 	} else if isKafkaQueue(sqsARN) {
 		kafkaN := serverConfig.Notify.GetKafkaByID(sqsARN.AccountID)
 		return (kafkaN.Enable && len(kafkaN.Brokers) > 0 &&
@@ -200,16 +216,14 @@ func validateQueueConfigs(queueConfigs []queueConfig) APIErrorCode {
 
 // Check all the queue configs for any duplicates.
 func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
-	var queueConfigARNS []string
+	queueConfigARNS := set.NewStringSet()
 
 	// Navigate through each configs and count the entries.
 	for _, config := range configs {
-		queueConfigARNS = append(queueConfigARNS, config.QueueARN)
+		queueConfigARNS.Add(config.QueueARN)
 	}
 
-	// Check if there are any duplicate counts.
-	if err := checkDuplicateStrings(queueConfigARNS); err != nil {
-		errorIf(err, "Invalid queue configs found.")
+	if len(queueConfigARNS) != len(configs) {
 		return ErrOverlappingConfigs
 	}
 
@@ -240,34 +254,50 @@ func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
 // Unmarshals input value of AWS ARN format into minioSqs object.
 // Returned value represents minio sqs types, currently supported are
 // - amqp
+// - mqtt
 // - nats
 // - elasticsearch
 // - redis
 // - postgresql
+// - mysql
 // - kafka
 // - webhook
 func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
-	mSqs = arnSQS{}
-	if !strings.HasPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":") {
-		return mSqs
+	strs := strings.SplitN(queueARN, ":", -1)
+	if len(strs) != 6 {
+		return
 	}
-	sqsType := strings.TrimPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":")
-	switch {
-	case hasSuffix(sqsType, queueTypeAMQP):
+	if serverConfig.GetRegion() != "" {
+		region := strs[3]
+		if region != serverConfig.GetRegion() {
+			return
+		}
+	}
+	sqsType := strs[5]
+	switch sqsType {
+	case queueTypeAMQP:
 		mSqs.Type = queueTypeAMQP
-	case hasSuffix(sqsType, queueTypeNATS):
+	case queueTypeMQTT:
+		mSqs.Type = queueTypeMQTT
+	case queueTypeNATS:
 		mSqs.Type = queueTypeNATS
-	case hasSuffix(sqsType, queueTypeElastic):
+	case queueTypeElastic:
 		mSqs.Type = queueTypeElastic
-	case hasSuffix(sqsType, queueTypeRedis):
+	case queueTypeRedis:
 		mSqs.Type = queueTypeRedis
-	case hasSuffix(sqsType, queueTypePostgreSQL):
+	case queueTypePostgreSQL:
 		mSqs.Type = queueTypePostgreSQL
-	case hasSuffix(sqsType, queueTypeKafka):
+	case queueTypeMySQL:
+		mSqs.Type = queueTypeMySQL
+	case queueTypeKafka:
 		mSqs.Type = queueTypeKafka
-	case hasSuffix(sqsType, queueTypeWebhook):
+	case queueTypeWebhook:
 		mSqs.Type = queueTypeWebhook
+	default:
+		errorIf(errors.New("invalid SQS type"), "SQS type: %s", sqsType)
 	} // Add more queues here.
-	mSqs.AccountID = strings.TrimSuffix(sqsType, ":"+mSqs.Type)
-	return mSqs
+
+	mSqs.AccountID = strs[4]
+
+	return
 }

cmd/bucket-notification-utils_test.go

@@ -259,11 +259,6 @@ func TestQueueARN(t *testing.T) {
 			queueARN: "arn:minio:sns:us-east-1:1:listen",
 			errCode:  ErrARNNotification,
 		},
-		// Invalid region 'us-west-1' in queue arn.
-		{
-			queueARN: "arn:minio:sqs:us-west-1:1:redis",
-			errCode:  ErrRegionNotification,
-		},
 		// Invalid queue name empty in queue arn.
 		{
 			queueARN: "arn:minio:sqs:us-east-1:1:",
@@ -298,6 +293,37 @@ func TestQueueARN(t *testing.T) {
 			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
 		}
 	}
+
+	// Test when server region is set.
+	rootPath, err = newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(rootPath)
+
+	testCases = []struct {
+		queueARN string
+		errCode  APIErrorCode
+	}{
+		// Incorrect region should produce error.
+		{
+			queueARN: "arn:minio:sqs:us-west-1:1:webhook",
+			errCode:  ErrRegionNotification,
+		},
+		// Correct region should not produce error.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:webhook",
+			errCode:  ErrNone,
+		},
+	}
+
+	// Validate all tests for queue arn.
+	for i, testCase := range testCases {
+		errCode := checkQueueARN(testCase.queueARN)
+		if testCase.errCode != errCode {
+			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
+		}
+	}
 }
 
 // Test unmarshal queue arn.
@@ -332,16 +358,16 @@ func TestUnmarshalSQSARN(t *testing.T) {
 			queueARN: "arn:minio:sqs:us-east-1:1:amqp",
 			Type:     "amqp",
 		},
+		// Valid mqtt queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:mqtt",
+			Type:     "mqtt",
+		},
 		// Invalid empty queue arn.
 		{
 			queueARN: "",
 			Type:     "",
 		},
-		// Invalid region 'us-west-1' in queue arn.
-		{
-			queueARN: "arn:minio:sqs:us-west-1:1:redis",
-			Type:     "",
-		},
 		// Partial queue arn.
 		{
 			queueARN: "arn:minio:sqs:",
@@ -361,4 +387,33 @@ func TestUnmarshalSQSARN(t *testing.T) {
 		}
 	}
 
+	// Test when the server region is set.
+	rootPath, err = newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(rootPath)
+
+	testCases = []struct {
+		queueARN string
+		Type     string
+	}{
+		// Incorrect region in ARN returns empty mSqs.Type
+		{
+			queueARN: "arn:minio:sqs:us-west-1:1:webhook",
+			Type:     "",
+		},
+		// Correct regionin ARN returns valid mSqs.Type
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:webhook",
+			Type:     "webhook",
+		},
+	}
+
+	for i, testCase := range testCases {
+		mSqs := unmarshalSqsARN(testCase.queueARN)
+		if testCase.Type != mSqs.Type {
+			t.Errorf("Test %d: Expected \"%s\", got \"%s\"", i+1, testCase.Type, mSqs.Type)
+		}
+	}
 }

cmd/bucket-policy-handlers_test.go

@@ -251,7 +251,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	initBucketPolicies(obj)
 
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucket(bucketName1); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName1, ""); err != nil {
 		t.Fatal(err)
 	}
 

cmd/certs.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
@@ -24,45 +25,34 @@ import (
 	"path/filepath"
 )
 
-// isSSL - returns true with both cert and key exists.
-func isSSL() bool {
-	return isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())
-}
-
-func parsePublicCertFile(certFile string) (certs []*x509.Certificate, err error) {
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(certFile); err != nil {
-		return certs, err
+func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
+	// Read certificate file.
+	var data []byte
+	if data, err = ioutil.ReadFile(certFile); err != nil {
+		return nil, err
 	}
 
 	// Parse all certs in the chain.
-	var block *pem.Block
-	var cert *x509.Certificate
-	current := bytes
+	current := data
 	for len(current) > 0 {
-		if block, current = pem.Decode(current); block == nil {
-			err = fmt.Errorf("Could not read PEM block from file %s", certFile)
-			return certs, err
+		var pemBlock *pem.Block
+		if pemBlock, current = pem.Decode(current); pemBlock == nil {
+			return nil, fmt.Errorf("Could not read PEM block from file %s", certFile)
 		}
 
-		if cert, err = x509.ParseCertificate(block.Bytes); err != nil {
-			return certs, err
+		var x509Cert *x509.Certificate
+		if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
+			return nil, err
 		}
 
-		certs = append(certs, cert)
+		x509Certs = append(x509Certs, x509Cert)
 	}
 
-	if len(certs) == 0 {
-		err = fmt.Errorf("Empty public certificate file %s", certFile)
+	if len(x509Certs) == 0 {
+		return nil, fmt.Errorf("Empty public certificate file %s", certFile)
 	}
 
-	return certs, err
-}
-
-// Reads certificate file and returns a list of parsed certificates.
-func readCertificateChain() ([]*x509.Certificate, error) {
-	return parsePublicCertFile(getPublicCertFile())
+	return x509Certs, nil
 }
 
 func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
@@ -91,7 +81,7 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	for _, caFile := range caFiles {
 		caCert, err := ioutil.ReadFile(caFile)
 		if err != nil {
-			return rootCAs, err
+			return nil, err
 		}
 
 		rootCAs.AppendCertsFromPEM(caCert)
@@ -100,9 +90,26 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	return rootCAs, nil
 }
 
-// loadRootCAs fetches CA files provided in minio config and adds them to globalRootCAs
-// Currently under Windows, there is no way to load system + user CAs at the same time
-func loadRootCAs() (err error) {
-	globalRootCAs, err = getRootCAs(getCADir())
-	return err
+func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
+	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
+		return nil, nil, nil, false, nil
+	}
+
+	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
+		return nil, nil, nil, false, err
+	}
+
+	var cert tls.Certificate
+	if cert, err = tls.LoadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
+		return nil, nil, nil, false, err
+	}
+
+	tlsCert = &cert
+
+	if rootCAs, err = getRootCAs(getCADir()); err != nil {
+		return nil, nil, nil, false, err
+	}
+
+	secureConn = true
+	return x509Certs, rootCAs, tlsCert, secureConn, nil
 }

cmd/checkport.go

@@ -1,64 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"net"
-	"os"
-	"syscall"
-)
-
-// checkPortAvailability - check if given port is already in use.
-// Note: The check method tries to listen on given port and closes it.
-// It is possible to have a disconnected client in this tiny window of time.
-func checkPortAvailability(port string) error {
-	network := [3]string{"tcp", "tcp4", "tcp6"}
-	for _, n := range network {
-		l, err := net.Listen(n, net.JoinHostPort("", port))
-		if err != nil {
-			if isAddrInUse(err) {
-				// Return error if another process is listening on the
-				// same port.
-				return err
-			}
-			// Ignore any other error (ex. EAFNOSUPPORT)
-			continue
-		}
-
-		// look for error so we don't have dangling connection
-		if err = l.Close(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// Return true if err is "address already in use" error.
-// syscall.EADDRINUSE is available on all OSes.
-func isAddrInUse(err error) bool {
-	if opErr, ok := err.(*net.OpError); ok {
-		if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
-			if errno, ok := sysErr.Err.(syscall.Errno); ok {
-				if errno == syscall.EADDRINUSE {
-					return true
-				}
-			}
-		}
-	}
-	return false
-}

cmd/checkport_test.go

@@ -1,54 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"net"
-	"runtime"
-	"testing"
-)
-
-// Tests for port availability logic written for server startup sequence.
-func TestCheckPortAvailability(t *testing.T) {
-	tests := []struct {
-		port string
-	}{
-		{getFreePort()},
-		{getFreePort()},
-	}
-	for _, test := range tests {
-		// This test should pass if the ports are available
-		err := checkPortAvailability(test.port)
-		if err != nil {
-			t.Fatalf("checkPortAvailability test failed for port: %s. Error: %v", test.port, err)
-		}
-
-		// Now use the ports and check again
-		ln, err := net.Listen("tcp", net.JoinHostPort("", test.port))
-		if err != nil {
-			t.Fail()
-		}
-		defer ln.Close()
-
-		err = checkPortAvailability(test.port)
-
-		// Skip if the os is windows due to https://github.com/golang/go/issues/7598
-		if err == nil && runtime.GOOS != globalWindowsOSName {
-			t.Fatalf("checkPortAvailability should fail for port: %s. Error: %v", test.port, err)
-		}
-	}
-}

cmd/common-main.go

@@ -0,0 +1,117 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/minio/cli"
+)
+
+// Check for updates and print a notification message
+func checkUpdate(mode string) {
+	// Its OK to ignore any errors during getUpdateInfo() here.
+	if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil {
+		if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" {
+			log.Println(updateMsg)
+		}
+	}
+}
+
+func enableLoggers() {
+	fileLogTarget := serverConfig.Logger.GetFile()
+	if fileLogTarget.Enable {
+		err := InitFileLogger(&fileLogTarget)
+		fatalIf(err, "Unable to initialize file logger")
+		log.AddTarget(fileLogTarget)
+	}
+
+	consoleLogTarget := serverConfig.Logger.GetConsole()
+	if consoleLogTarget.Enable {
+		InitConsoleLogger(&consoleLogTarget)
+	}
+
+	log.SetConsoleTarget(consoleLogTarget)
+}
+
+func initConfig() {
+	// Config file does not exist, we create it fresh and return upon success.
+	if isFile(getConfigFile()) {
+		fatalIf(migrateConfig(), "Config migration failed.")
+		fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19)
+	} else {
+		fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
+		log.Println("Created minio configuration file successfully at " + getConfigDir())
+	}
+}
+
+func handleCommonCmdArgs(ctx *cli.Context) {
+	// Set configuration directory.
+	{
+		// Get configuration directory from command line argument.
+		configDir := ctx.String("config-dir")
+		if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
+			configDir = ctx.GlobalString("config-dir")
+		}
+		if configDir == "" {
+			fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
+		}
+
+		// Disallow relative paths, figure out absolute paths.
+		configDirAbs, err := filepath.Abs(configDir)
+		fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
+
+		setConfigDir(configDirAbs)
+	}
+}
+
+func handleCommonEnvVars() {
+	// Start profiler if env is set.
+	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
+		globalProfiler = startProfiler(profiler)
+	}
+
+	// Check if object cache is disabled.
+	globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
+
+	accessKey := os.Getenv("MINIO_ACCESS_KEY")
+	secretKey := os.Getenv("MINIO_SECRET_KEY")
+	if accessKey != "" && secretKey != "" {
+		cred, err := createCredential(accessKey, secretKey)
+		fatalIf(err, "Invalid access/secret Key set in environment.")
+
+		// credential Envs are set globally.
+		globalIsEnvCreds = true
+		globalActiveCred = cred
+	}
+
+	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
+		browserFlag, err := ParseBrowserFlag(browser)
+		if err != nil {
+			fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
+		}
+
+		// browser Envs are set globally, this does not represent
+		// if browser is turned off or on.
+		globalIsEnvBrowser = true
+		globalIsBrowserEnabled = bool(browserFlag)
+	}
+}

cmd/config-dir.go

@@ -21,7 +21,6 @@ import (
 	"sync"
 
 	homedir "github.com/minio/go-homedir"
-	"github.com/minio/mc/pkg/console"
 )
 
 const (
@@ -97,9 +96,7 @@ func (config *ConfigDir) GetPrivateKeyFile() string {
 
 func mustGetDefaultConfigDir() string {
 	homeDir, err := homedir.Dir()
-	if err != nil {
-		console.Fatalln("Unable to get home directory.", err)
-	}
+	fatalIf(err, "Unable to get home directory.")
 
 	return filepath.Join(homeDir, defaultMinioConfigDir)
 }
@@ -133,7 +130,3 @@ func getPublicCertFile() string {
 func getPrivateKeyFile() string {
 	return configDir.GetPrivateKeyFile()
 }
-
-func isConfigFileExists() bool {
-	return isFile(getConfigFile())
-}

cmd/config-migrate_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -45,12 +46,12 @@ func TestServerConfigMigrateV1(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Check if config v1 is removed from filesystem
-	if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
+	if _, err := osStat(configPath); err == nil || !os.IsNotExist(err) {
 		t.Fatal("Config V1 file is not purged")
 	}
 
 	// Initialize server config and check again if everything is fine
-	if err := loadConfig(envParams{}); err != nil {
+	if err := loadConfig(); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 }
@@ -109,10 +110,26 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	if err := migrateV13ToV14(); err != nil {
 		t.Fatal("migrate v13 to v14 should succeed when no config file is found")
 	}
+	if err := migrateV14ToV15(); err != nil {
+		t.Fatal("migrate v14 to v15 should succeed when no config file is found")
+	}
+	if err := migrateV15ToV16(); err != nil {
+		t.Fatal("migrate v15 to v16 should succeed when no config file is found")
+	}
+	if err := migrateV16ToV17(); err != nil {
+		t.Fatal("migrate v16 to v17 should succeed when no config file is found")
+	}
+	if err := migrateV17ToV18(); err != nil {
+		t.Fatal("migrate v17 to v18 should succeed when no config file is found")
+	}
+	if err := migrateV18ToV19(); err != nil {
+		t.Fatal("migrate v18 to v19 should succeed when no config file is found")
+	}
+
 }
 
-// Test if a config migration from v2 to v12 is successfully done
-func TestServerConfigMigrateV2toV14(t *testing.T) {
+// Test if a config migration from v2 to v19 is successfully done
+func TestServerConfigMigrateV2toV19(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Init Test config failed")
@@ -140,18 +157,19 @@ func TestServerConfigMigrateV2toV14(t *testing.T) {
 	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
+
 	// Fire a migrateConfig()
 	if err := migrateConfig(); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Initialize server config and check again if everything is fine
-	if err := loadConfig(envParams{}); err != nil {
+	if err := loadConfig(); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 
 	// Check the version number in the upgraded config file
-	expectedVersion := v14
+	expectedVersion := v19
 	if serverConfig.Version != expectedVersion {
 		t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
 	}
@@ -178,7 +196,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	configPath := rootPath + "/" + minioConfigFile
 
 	// Create a corrupted config file
-	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\""), 0644); err != nil {
+	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -219,4 +237,55 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV13ToV14(); err == nil {
 		t.Fatal("migrateConfigV13ToV14() should fail with a corrupted json")
 	}
+	if err := migrateV14ToV15(); err == nil {
+		t.Fatal("migrateConfigV14ToV15() should fail with a corrupted json")
+	}
+	if err := migrateV15ToV16(); err == nil {
+		t.Fatal("migrateConfigV15ToV16() should fail with a corrupted json")
+	}
+	if err := migrateV16ToV17(); err == nil {
+		t.Fatal("migrateConfigV16ToV17() should fail with a corrupted json")
+	}
+	if err := migrateV17ToV18(); err == nil {
+		t.Fatal("migrateConfigV17ToV18() should fail with a corrupted json")
+	}
+	if err := migrateV18ToV19(); err == nil {
+		t.Fatal("migrateConfigV18ToV19() should fail with a corrupted json")
+	}
+}
+
+// Test if all migrate code returns error with corrupted config files
+func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
+	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	setConfigDir(rootPath)
+	configPath := rootPath + "/" + minioConfigFile
+
+	for i := 3; i <= 17; i++ {
+		// Create a corrupted config file
+		if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
+			0644); err != nil {
+			t.Fatal("Unexpected error: ", err)
+		}
+
+		// Test different migrate versions and be sure they are returning an error
+		if err = migrateConfig(); err == nil {
+			t.Fatal("migrateConfig() should fail with a corrupted json")
+		}
+	}
+
+	// Create a corrupted config file for version '2'.
+	if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0644); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
+	// Test different migrate versions and be sure they are returning an error
+	if err = migrateConfig(); err == nil {
+		t.Fatal("migrateConfig() should fail with a corrupted json")
+	}
 }

cmd/config-old.go

@@ -16,30 +16,7 @@
 
 package cmd
 
-import (
-	"os"
-	"path/filepath"
-	"sync"
-
-	"github.com/minio/minio/pkg/quick"
-)
-
-func loadOldConfig(configFile string, config interface{}) (interface{}, error) {
-	if _, err := os.Stat(configFile); err != nil {
-		return nil, err
-	}
-
-	qc, err := quick.New(config)
-	if err != nil {
-		return nil, err
-	}
-
-	if err = qc.Load(configFile); err != nil {
-		return nil, err
-	}
-
-	return config, nil
-}
+import "sync"
 
 /////////////////// Config V1 ///////////////////
 type configV1 struct {
@@ -48,16 +25,6 @@ type configV1 struct {
 	SecretKey string `json:"secretAccessKey"`
 }
 
-// loadConfigV1 load config
-func loadConfigV1() (*configV1, error) {
-	configFile := filepath.Join(getConfigDir(), "fsUsers.json")
-	config, err := loadOldConfig(configFile, &configV1{Version: "1"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV1), err
-}
-
 /////////////////// Config V2 ///////////////////
 type configV2 struct {
 	Version     string `json:"version"`
@@ -80,18 +47,7 @@ type configV2 struct {
 	} `json:"fileLogger"`
 }
 
-// loadConfigV2 load config version '2'.
-func loadConfigV2() (*configV2, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &configV2{Version: "2"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV2), err
-}
-
 /////////////////// Config V3 ///////////////////
-
 // backendV3 type.
 type backendV3 struct {
 	Type  string   `json:"type"`
@@ -143,16 +99,6 @@ type configV3 struct {
 	Logger loggerV3 `json:"logger"`
 }
 
-// loadConfigV3 load config version '3'.
-func loadConfigV3() (*configV3, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &configV3{Version: "3"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV3), err
-}
-
 // logger type representing version '4' logger config.
 type loggerV4 struct {
 	Console struct {
@@ -183,16 +129,6 @@ type configV4 struct {
 	Logger loggerV4 `json:"logger"`
 }
 
-// loadConfigV4 load config version '4'.
-func loadConfigV4() (*configV4, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &configV4{Version: "4"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV4), err
-}
-
 // logger type representing version '5' logger config.
 type loggerV5 struct {
 	Console struct {
@@ -250,20 +186,22 @@ type configV5 struct {
 	Logger loggerV5 `json:"logger"`
 }
 
-// loadConfigV5 load config version '5'.
-func loadConfigV5() (*configV5, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &configV5{Version: "5"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV5), err
+// consoleLogger - default logger if not other logging is enabled.
+type consoleLoggerV1 struct {
+	Enable bool   `json:"enable"`
+	Level  string `json:"level"`
+}
+
+type fileLoggerV1 struct {
+	Enable   bool   `json:"enable"`
+	Filename string `json:"fileName"`
+	Level    string `json:"level"`
 }
 
 type loggerV6 struct {
-	Console consoleLogger  `json:"console"`
-	File    fileLogger     `json:"file"`
-	Syslog  syslogLoggerV3 `json:"syslog"`
+	Console consoleLoggerV1 `json:"console"`
+	File    fileLoggerV1    `json:"file"`
+	Syslog  syslogLoggerV3  `json:"syslog"`
 }
 
 // configV6 server configuration version '6'.
@@ -281,16 +219,6 @@ type configV6 struct {
 	Notify notifierV1 `json:"notify"`
 }
 
-// loadConfigV6 load config version '6'.
-func loadConfigV6() (*configV6, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &configV6{Version: "6"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV6), err
-}
-
 // Notifier represents collection of supported notification queues in version
 // 1 without NATS streaming.
 type notifierV1 struct {
@@ -331,16 +259,6 @@ type serverConfigV7 struct {
 	rwMutex *sync.RWMutex
 }
 
-// loadConfigV7 load config version '7'.
-func loadConfigV7() (*serverConfigV7, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV7{Version: "7"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV7), err
-}
-
 // serverConfigV8 server configuration version '8'. Adds NATS notifier
 // configuration.
 type serverConfigV8 struct {
@@ -360,16 +278,6 @@ type serverConfigV8 struct {
 	rwMutex *sync.RWMutex
 }
 
-// loadConfigV8 load config version '8'.
-func loadConfigV8() (*serverConfigV8, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV8{Version: "8"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV8), err
-}
-
 // serverConfigV9 server configuration version '9'. Adds PostgreSQL
 // notifier configuration.
 type serverConfigV9 struct {
@@ -389,13 +297,10 @@ type serverConfigV9 struct {
 	rwMutex *sync.RWMutex
 }
 
-func loadConfigV9() (*serverConfigV9, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV9{Version: "9"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV9), err
+type loggerV7 struct {
+	sync.RWMutex
+	Console consoleLoggerV1 `json:"console"`
+	File    fileLoggerV1    `json:"file"`
 }
 
 // serverConfigV10 server configuration version '10' which is like
@@ -409,21 +314,12 @@ type serverConfigV10 struct {
 	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
-	Logger logger `json:"logger"`
+	Logger loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
 	Notify notifierV1 `json:"notify"`
 }
 
-func loadConfigV10() (*serverConfigV10, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV10{Version: "10"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV10), err
-}
-
 // natsNotifyV1 - structure was valid until config V 11
 type natsNotifyV1 struct {
 	Enable       bool   `json:"enable"`
@@ -446,21 +342,12 @@ type serverConfigV11 struct {
 	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
-	Logger logger `json:"logger"`
+	Logger loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
 	Notify notifierV1 `json:"notify"`
 }
 
-func loadConfigV11() (*serverConfigV11, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV11{Version: "11"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV11), err
-}
-
 // serverConfigV12 server configuration version '12' which is like
 // version '11' except it adds support for NATS streaming notifications.
 type serverConfigV12 struct {
@@ -471,21 +358,12 @@ type serverConfigV12 struct {
 	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
-	Logger logger `json:"logger"`
+	Logger loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
 	Notify notifierV2 `json:"notify"`
 }
 
-func loadConfigV12() (*serverConfigV12, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV12{Version: "12"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV12), err
-}
-
 // serverConfigV13 server configuration version '13' which is like
 // version '12' except it adds support for webhook notification.
 type serverConfigV13 struct {
@@ -496,17 +374,97 @@ type serverConfigV13 struct {
 	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
-	Logger *logger `json:"logger"`
+	Logger *loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
 	Notify *notifier `json:"notify"`
 }
 
-func loadConfigV13() (*serverConfigV13, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV13{Version: "13"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV13), err
+// serverConfigV14 server configuration version '14' which is like
+// version '13' except it adds support of browser param.
+type serverConfigV14 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggerV7 `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV15 server configuration version '15' which is like
+// version '14' except it adds mysql support
+type serverConfigV15 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggerV7 `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV16 server configuration version '16' which is like
+// version '15' except it makes a change to logging configuration.
+type serverConfigV16 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV17 server configuration version '17' which is like
+// version '16' except it adds support for "format" parameter in
+// database event notification targets: PostgreSQL, MySQL, Redis and
+// Elasticsearch.
+type serverConfigV17 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV18 server configuration version '18' which is like
+// version '17' except it adds support for "deliveryMode" parameter in
+// the AMQP notification target.
+type serverConfigV18 struct {
+	sync.RWMutex
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
 }

cmd/config-v19.go

@@ -20,55 +20,124 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"os"
-	"strings"
 	"sync"
 
 	"github.com/minio/minio/pkg/quick"
 	"github.com/tidwall/gjson"
 )
 
-// Read Write mutex for safe access to ServerConfig.
-var serverConfigMu sync.RWMutex
-
 // Config version
-var v14 = "14"
+const v19 = "19"
 
-// serverConfigV14 server configuration version '14' which is like
-// version '13' except it adds support of browser param.
-type serverConfigV14 struct {
+var (
+	// serverConfig server config.
+	serverConfig   *serverConfigV19
+	serverConfigMu sync.RWMutex
+)
+
+// serverConfigV19 server configuration version '19' which is like
+// version '18' except it adds support for MQTT notifications.
+type serverConfigV19 struct {
+	sync.RWMutex
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
-	Browser    string     `json:"browser"`
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
 
 	// Additional error logging configuration.
-	Logger *logger `json:"logger"`
+	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
 	Notify *notifier `json:"notify"`
 }
 
-func newServerConfigV14() *serverConfigV14 {
-	srvCfg := &serverConfigV14{
-		Version: v14,
-		Region:  globalMinioDefaultRegion,
-		Logger:  &logger{},
-		Notify:  &notifier{},
+// GetVersion get current config version.
+func (s *serverConfigV19) GetVersion() string {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.Version
+}
+
+// SetRegion set new region.
+func (s *serverConfigV19) SetRegion(region string) {
+	s.Lock()
+	defer s.Unlock()
+
+	s.Region = region
+}
+
+// GetRegion get current region.
+func (s *serverConfigV19) GetRegion() string {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.Region
+}
+
+// SetCredentials set new credentials.
+func (s *serverConfigV19) SetCredential(creds credential) {
+	s.Lock()
+	defer s.Unlock()
+
+	// Set updated credential.
+	s.Credential = creds
+}
+
+// GetCredentials get current credentials.
+func (s *serverConfigV19) GetCredential() credential {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.Credential
+}
+
+// SetBrowser set if browser is enabled.
+func (s *serverConfigV19) SetBrowser(b bool) {
+	s.Lock()
+	defer s.Unlock()
+
+	// Set the new value.
+	s.Browser = BrowserFlag(b)
+}
+
+// GetCredentials get current credentials.
+func (s *serverConfigV19) GetBrowser() bool {
+	s.RLock()
+	defer s.RUnlock()
+
+	return bool(s.Browser)
+}
+
+// Save config.
+func (s *serverConfigV19) Save() error {
+	s.RLock()
+	defer s.RUnlock()
+
+	// Save config file.
+	return quick.Save(getConfigFile(), s)
+}
+
+func newServerConfigV19() *serverConfigV19 {
+	srvCfg := &serverConfigV19{
+		Version:    v19,
+		Credential: mustGetNewCredential(),
+		Region:     globalMinioDefaultRegion,
+		Browser:    true,
+		Logger:     &loggers{},
+		Notify:     &notifier{},
 	}
-	srvCfg.SetCredential(mustGetNewCredential())
-	srvCfg.SetBrowser("on")
+
 	// Enable console logger by default on a fresh run.
-	srvCfg.Logger.Console = consoleLogger{
-		Enable: true,
-		Level:  "error",
-	}
+	srvCfg.Logger.Console = NewConsoleLogger()
 
 	// Make sure to initialize notification configs.
 	srvCfg.Notify.AMQP = make(map[string]amqpNotify)
 	srvCfg.Notify.AMQP["1"] = amqpNotify{}
+	srvCfg.Notify.MQTT = make(map[string]mqttNotify)
+	srvCfg.Notify.MQTT["1"] = mqttNotify{}
 	srvCfg.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
 	srvCfg.Notify.ElasticSearch["1"] = elasticSearchNotify{}
 	srvCfg.Notify.Redis = make(map[string]redisNotify)
@@ -77,6 +146,8 @@ func newServerConfigV14() *serverConfigV14 {
 	srvCfg.Notify.NATS["1"] = natsNotify{}
 	srvCfg.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
 	srvCfg.Notify.PostgreSQL["1"] = postgreSQLNotify{}
+	srvCfg.Notify.MySQL = make(map[string]mySQLNotify)
+	srvCfg.Notify.MySQL["1"] = mySQLNotify{}
 	srvCfg.Notify.Kafka = make(map[string]kafkaNotify)
 	srvCfg.Notify.Kafka["1"] = kafkaNotify{}
 	srvCfg.Notify.Webhook = make(map[string]webhookNotify)
@@ -87,22 +158,21 @@ func newServerConfigV14() *serverConfigV14 {
 
 // newConfig - initialize a new server config, saves env parameters if
 // found, otherwise use default parameters
-func newConfig(envParams envParams) error {
+func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfigV14()
+	srvCfg := newServerConfigV19()
 
-	// If env is set for a fresh start, save them to config file.
+	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
-		srvCfg.SetCredential(envParams.creds)
+		srvCfg.SetCredential(globalActiveCred)
 	}
 
 	if globalIsEnvBrowser {
-		srvCfg.SetBrowser(envParams.browser)
+		srvCfg.SetBrowser(globalIsBrowserEnabled)
 	}
 
-	// Create config path.
-	if err := createConfigDir(); err != nil {
-		return err
+	if globalIsEnvRegion {
+		srvCfg.SetRegion(globalServerRegion)
 	}
 
 	// hold the mutex lock before a new config is assigned.
@@ -116,51 +186,6 @@ func newConfig(envParams envParams) error {
 	return serverConfig.Save()
 }
 
-// loadConfig - loads a new config from disk, overrides params from env
-// if found and valid
-func loadConfig(envParams envParams) error {
-	configFile := getConfigFile()
-	if _, err := os.Stat(configFile); err != nil {
-		return err
-	}
-
-	srvCfg := &serverConfigV14{}
-
-	qc, err := quick.New(srvCfg)
-	if err != nil {
-		return err
-	}
-
-	if err = qc.Load(configFile); err != nil {
-		return err
-	}
-
-	// If env is set override the credentials from config file.
-	if globalIsEnvCreds {
-		srvCfg.SetCredential(envParams.creds)
-	}
-
-	if globalIsEnvBrowser {
-		srvCfg.SetBrowser(envParams.browser)
-	}
-
-	if strings.ToLower(srvCfg.GetBrowser()) == "off" {
-		globalIsBrowserEnabled = false
-	}
-
-	// hold the mutex lock before a new config is assigned.
-	serverConfigMu.Lock()
-	// Save the loaded config globally.
-	serverConfig = srvCfg
-	serverConfigMu.Unlock()
-
-	if serverConfig.Version != v14 {
-		return errors.New("Unsupported config version `" + serverConfig.Version + "`.")
-	}
-
-	return nil
-}
-
 // doCheckDupJSONKeys recursively detects duplicate json keys
 func doCheckDupJSONKeys(key, value gjson.Result) error {
 	// Key occurrences map of the current scope to count
@@ -212,159 +237,86 @@ func checkDupJSONKeys(json string) error {
 	return doCheckDupJSONKeys(rootKey, config)
 }
 
-// validateConfig checks for
-func validateConfig() error {
+// getValidConfig - returns valid server configuration
+func getValidConfig() (*serverConfigV19, error) {
+	srvCfg := &serverConfigV19{
+		Region:  globalMinioDefaultRegion,
+		Browser: true,
+	}
 
-	// Get file config path
 	configFile := getConfigFile()
-
-	srvCfg := &serverConfigV14{}
-
-	// Load config file
-	qc, err := quick.New(srvCfg)
-	if err != nil {
-		return err
-	}
-	if err = qc.Load(configFile); err != nil {
-		return err
+	if _, err := quick.Load(configFile, srvCfg); err != nil {
+		return nil, err
 	}
 
-	// Check if config version is valid
-	if srvCfg.GetVersion() != v14 {
-		return errors.New("bad config version, expected: " + v14)
+	if srvCfg.Version != v19 {
+		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v19, srvCfg.Version)
 	}
 
 	// Load config file json and check for duplication json keys
 	jsonBytes, err := ioutil.ReadFile(configFile)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	if err := checkDupJSONKeys(string(jsonBytes)); err != nil {
-		return err
+	if err = checkDupJSONKeys(string(jsonBytes)); err != nil {
+		return nil, err
 	}
 
-	// Validate region field
-	if srvCfg.GetRegion() == "" {
-		return errors.New("Region config value cannot be empty")
-	}
+	// Validate credential fields only when
+	// they are not set via the environment
 
-	// Validate browser field
-	if b := strings.ToLower(srvCfg.GetBrowser()); b != "on" && b != "off" {
-		return fmt.Errorf("Browser config value %s is invalid", b)
-	}
-
-	// Validate credential field
-	if !srvCfg.Credential.IsValid() {
-		return errors.New("invalid credential")
+	// Error out if global is env credential is not set and config has invalid credential
+	if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
+		return nil, errors.New("invalid credential in config file " + configFile)
 	}
 
 	// Validate logger field
-	if err := srvCfg.Logger.Validate(); err != nil {
-		return err
+	if err = srvCfg.Logger.Validate(); err != nil {
+		return nil, err
 	}
 
 	// Validate notify field
-	if err := srvCfg.Notify.Validate(); err != nil {
-		return err
+	if err = srvCfg.Notify.Validate(); err != nil {
+		return nil, err
 	}
 
-	return nil
+	return srvCfg, nil
 }
 
-// serverConfig server config.
-var serverConfig *serverConfigV14
-
-// GetVersion get current config version.
-func (s serverConfigV14) GetVersion() string {
-	serverConfigMu.RLock()
-	defer serverConfigMu.RUnlock()
-
-	return s.Version
-}
-
-// SetRegion set new region.
-func (s *serverConfigV14) SetRegion(region string) {
-	serverConfigMu.Lock()
-	defer serverConfigMu.Unlock()
-
-	// Empty region means "us-east-1" by default from S3 spec.
-	if region == "" {
-		region = "us-east-1"
-	}
-	s.Region = region
-}
-
-// GetRegion get current region.
-func (s serverConfigV14) GetRegion() string {
-	serverConfigMu.RLock()
-	defer serverConfigMu.RUnlock()
-
-	if s.Region != "" {
-		return s.Region
-	} // region empty
-
-	// Empty region means "us-east-1" by default from S3 spec.
-	return "us-east-1"
-}
-
-// SetCredentials set new credentials.
-func (s *serverConfigV14) SetCredential(creds credential) {
-	serverConfigMu.Lock()
-	defer serverConfigMu.Unlock()
-
-	// Set updated credential.
-	s.Credential = creds
-}
-
-// GetCredentials get current credentials.
-func (s serverConfigV14) GetCredential() credential {
-	serverConfigMu.RLock()
-	defer serverConfigMu.RUnlock()
-
-	return s.Credential
-}
-
-// SetBrowser set if browser is enabled.
-func (s *serverConfigV14) SetBrowser(v string) {
-	serverConfigMu.Lock()
-	defer serverConfigMu.Unlock()
-
-	// Set browser param
-	if v == "" {
-		v = "on" // Browser is on by default.
-	}
-
-	// Set the new value.
-	s.Browser = v
-}
-
-// GetCredentials get current credentials.
-func (s serverConfigV14) GetBrowser() string {
-	serverConfigMu.RLock()
-	defer serverConfigMu.RUnlock()
-
-	if s.Browser != "" {
-		return s.Browser
-	} // empty browser.
-
-	// Empty browser means "on" by default.
-	return "on"
-}
-
-// Save config.
-func (s serverConfigV14) Save() error {
-	serverConfigMu.RLock()
-	defer serverConfigMu.RUnlock()
-
-	// get config file.
-	configFile := getConfigFile()
-
-	// initialize quick.
-	qc, err := quick.New(&s)
+// loadConfig - loads a new config from disk, overrides params from env
+// if found and valid
+func loadConfig() error {
+	srvCfg, err := getValidConfig()
 	if err != nil {
 		return err
 	}
 
-	// Save config file.
-	return qc.Save(configFile)
+	// If env is set override the credentials from config file.
+	if globalIsEnvCreds {
+		srvCfg.SetCredential(globalActiveCred)
+	}
+
+	if globalIsEnvBrowser {
+		srvCfg.SetBrowser(globalIsBrowserEnabled)
+	}
+
+	if globalIsEnvRegion {
+		srvCfg.SetRegion(globalServerRegion)
+	}
+
+	// hold the mutex lock before a new config is assigned.
+	serverConfigMu.Lock()
+	serverConfig = srvCfg
+	if !globalIsEnvCreds {
+		globalActiveCred = serverConfig.GetCredential()
+	}
+	if !globalIsEnvBrowser {
+		globalIsBrowserEnabled = serverConfig.GetBrowser()
+	}
+	if !globalIsEnvRegion {
+		globalServerRegion = serverConfig.GetRegion()
+	}
+	serverConfigMu.Unlock()
+
+	return nil
 }

cmd/config-v19_test.go

@@ -76,38 +76,49 @@ func TestServerConfig(t *testing.T) {
 	serverConfig.Notify.SetWebhookByID("2", webhookNotify{})
 	savedNotifyCfg5 := serverConfig.Notify.GetWebhookByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg5, webhookNotify{}) {
-		t.Errorf("Expecting Webhook config %#v found %#v", webhookNotify{}, savedNotifyCfg3)
+		t.Errorf("Expecting Webhook config %#v found %#v", webhookNotify{}, savedNotifyCfg5)
 	}
 
 	// Set new console logger.
-	serverConfig.Logger.SetConsole(consoleLogger{
-		Enable: true,
-	})
+	// Set new MySQL notification id.
+	serverConfig.Notify.SetMySQLByID("2", mySQLNotify{})
+	savedNotifyCfg6 := serverConfig.Notify.GetMySQLByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg6, mySQLNotify{}) {
+		t.Errorf("Expecting Webhook config %#v found %#v", mySQLNotify{}, savedNotifyCfg6)
+	}
+
+	// Set new console logger.
+	// Set new MQTT notification id.
+	serverConfig.Notify.SetMQTTByID("2", mqttNotify{})
+	savedNotifyCfg7 := serverConfig.Notify.GetMQTTByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg7, mqttNotify{}) {
+		t.Errorf("Expecting Webhook config %#v found %#v", mqttNotify{}, savedNotifyCfg7)
+	}
+
+	consoleLogger := NewConsoleLogger()
+	serverConfig.Logger.SetConsole(consoleLogger)
 	consoleCfg := serverConfig.Logger.GetConsole()
-	if !reflect.DeepEqual(consoleCfg, consoleLogger{Enable: true}) {
-		t.Errorf("Expecting console logger config %#v found %#v", consoleLogger{Enable: true}, consoleCfg)
+	if !reflect.DeepEqual(consoleCfg, consoleLogger) {
+		t.Errorf("Expecting console logger config %#v found %#v", consoleLogger, consoleCfg)
 	}
 	// Set new console logger.
-	serverConfig.Logger.SetConsole(consoleLogger{
-		Enable: false,
-	})
+	consoleLogger.Enable = false
+	serverConfig.Logger.SetConsole(consoleLogger)
 
 	// Set new file logger.
-	serverConfig.Logger.SetFile(fileLogger{
-		Enable: true,
-	})
+	fileLogger := NewFileLogger("test-log-file")
+	serverConfig.Logger.SetFile(fileLogger)
 	fileCfg := serverConfig.Logger.GetFile()
-	if !reflect.DeepEqual(fileCfg, fileLogger{Enable: true}) {
-		t.Errorf("Expecting file logger config %#v found %#v", fileLogger{Enable: true}, consoleCfg)
+	if !reflect.DeepEqual(fileCfg, fileLogger) {
+		t.Errorf("Expecting file logger config %#v found %#v", fileLogger, fileCfg)
 	}
 	// Set new file logger.
-	serverConfig.Logger.SetFile(fileLogger{
-		Enable: false,
-	})
+	fileLogger.Enable = false
+	serverConfig.Logger.SetFile(fileLogger)
 
 	// Match version.
-	if serverConfig.GetVersion() != v14 {
-		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v14)
+	if serverConfig.GetVersion() != v19 {
+		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v19)
 	}
 
 	// Attempt to save.
@@ -119,7 +130,7 @@ func TestServerConfig(t *testing.T) {
 	setConfigDir(rootPath)
 
 	// Initialize server config.
-	if err := loadConfig(envParams{}); err != nil {
+	if err := loadConfig(); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 }
@@ -135,10 +146,10 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	os.Setenv("MINIO_SECRET_KEY", "minio123")
 	defer os.Unsetenv("MINIO_SECRET_KEY")
 
-	defer func() {
-		globalIsEnvBrowser = false
-		globalIsEnvCreds = false
-	}()
+	os.Setenv("MINIO_REGION", "us-west-1")
+	defer os.Unsetenv("MINIO_REGION")
+
+	defer resetGlobalIsEnvs()
 
 	// Get test root.
 	rootPath, err := getTestRoot()
@@ -146,6 +157,8 @@ func TestServerConfigWithEnvs(t *testing.T) {
 		t.Error(err)
 	}
 
+	serverHandleEnvVars()
+
 	// Do this only once here.
 	setConfigDir(rootPath)
 
@@ -156,8 +169,13 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	defer removeAll(rootPath)
 
 	// Check if serverConfig has
-	if serverConfig.GetBrowser() != "off" {
-		t.Errorf("Expecting browser `off` found %s", serverConfig.GetBrowser())
+	if serverConfig.GetBrowser() {
+		t.Errorf("Expecting browser is set to false found %v", serverConfig.GetBrowser())
+	}
+
+	// Check if serverConfig has
+	if serverConfig.GetRegion() != "us-west-1" {
+		t.Errorf("Expecting region to be \"us-west-1\" found %v", serverConfig.GetRegion())
 	}
 
 	// Check if serverConfig has
@@ -170,6 +188,7 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	if cred.SecretKey != "minio123" {
 		t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
 	}
+
 }
 
 func TestCheckDupJSONKeys(t *testing.T) {
@@ -212,7 +231,7 @@ func TestValidateConfig(t *testing.T) {
 
 	configPath := filepath.Join(rootPath, minioConfigFile)
 
-	v := v14
+	v := v19
 
 	testCases := []struct {
 		configData string
@@ -248,39 +267,66 @@ func TestValidateConfig(t *testing.T) {
 		// Test 10 - duplicated json keys
 		{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
 
-		// Test 11 - Wrong Console logger level
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "console": { "enable": true, "level": "foo" } }}`, false},
+		// Test 11 - empty filename field in File
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "filename": "" } }}`, false},
 
-		// Test 12 - Wrong File logger level
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "level": "foo" } }}`, false},
-
-		// Test 13 - Test AMQP
+		// Test 12 - Test AMQP
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
 
-		// Test 14 - Test NATS
+		// Test 13 - Test NATS
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
 
-		// Test 15 - Test ElasticSearch
+		// Test 14 - Test ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
 
-		// Test 16 - Test Redis
+		// Test 15 - Test Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
 
-		// Test 17 - Test PostgreSQL
+		// Test 16 - Test PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
-		// Test 18 - Test Kafka
+		// Test 17 - Test Kafka
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
 
-		// Test 19 - Test Webhook
+		// Test 18 - Test Webhook
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
+
+		// Test 20 - Test MySQL
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
+
+		// Test 21 - Test Format for MySQL
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
+
+		// Test 22 - Test valid Format for MySQL
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+
+		// Test 23 - Test Format for PostgreSQL
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
+
+		// Test 24 - Test valid Format for PostgreSQL
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+
+		// Test 25 - Test Format for ElasticSearch
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
+
+		// Test 26 - Test valid Format for ElasticSearch
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
+
+		// Test 27 - Test Format for Redis
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
+
+		// Test 28 - Test valid Format for Redis
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
+
+		// Test 29 - Test MQTT
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
 	}
 
 	for i, testCase := range testCases {
 		if werr := ioutil.WriteFile(configPath, []byte(testCase.configData), 0700); werr != nil {
 			t.Fatal(werr)
 		}
-		verr := validateConfig()
+		_, verr := getValidConfig()
 		if testCase.shouldPass && verr != nil {
 			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, verr)
 		}

cmd/console-logger.go

@@ -0,0 +1,74 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// ConsoleLogger - console logger which logs into stderr.
+type ConsoleLogger struct {
+	BaseLogTarget
+}
+
+// Fire - log entry handler.
+func (logger ConsoleLogger) Fire(entry *logrus.Entry) error {
+	if !logger.Enable {
+		return nil
+	}
+
+	msgBytes, err := logger.formatter.Format(entry)
+	if err == nil {
+		fmt.Fprintf(os.Stderr, string(msgBytes))
+	}
+
+	return err
+}
+
+// String - represents ConsoleLogger as string.
+func (logger ConsoleLogger) String() string {
+	enableStr := "disabled"
+	if logger.Enable {
+		enableStr = "enabled"
+	}
+
+	return fmt.Sprintf("console:%s", enableStr)
+}
+
+// NewConsoleLogger - return new console logger object.
+func NewConsoleLogger() (logger ConsoleLogger) {
+	logger.Enable = true
+	logger.formatter = new(logrus.TextFormatter)
+
+	return logger
+}
+
+// InitConsoleLogger - initializes console logger.
+func InitConsoleLogger(logger *ConsoleLogger) {
+	if !logger.Enable {
+		return
+	}
+
+	if logger.formatter == nil {
+		logger.formatter = new(logrus.TextFormatter)
+	}
+
+	return
+}

cmd/credential.go

@@ -21,56 +21,46 @@ import (
 	"encoding/base64"
 	"errors"
 
-	"github.com/minio/mc/pkg/console"
-
 	"golang.org/x/crypto/bcrypt"
 )
 
 const (
-	accessKeyMinLen       = 5
-	accessKeyMaxLen       = 20
-	secretKeyMinLen       = 8
-	secretKeyMaxLenAmazon = 40
-	alphaNumericTable     = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-	alphaNumericTableLen  = byte(len(alphaNumericTable))
+	// Minimum length for Minio access key.
+	accessKeyMinLen = 5
+
+	// Maximum length for Minio access key.
+	// There is no max length enforcement for access keys
+	accessKeyMaxLen = 20
+
+	// Minimum length for Minio secret key for both server and gateway mode.
+	secretKeyMinLen = 8
+
+	// Maximum secret key length for Minio, this
+	// is used when autogenerating new credentials.
+	// There is no max length enforcement for secret keys
+	secretKeyMaxLen = 40
+
+	// Alpha numeric table used for generating access keys.
+	alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+	// Total length of the alpha numeric table.
+	alphaNumericTableLen = byte(len(alphaNumericTable))
 )
 
+// Common errors generated for access and secret key validation.
 var (
-	errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be 5 to 20 characters in length")
-	errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be 8 to 40 characters in length")
+	errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be minimum 5 characters in length")
+	errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be minimum 8 characters in length")
 )
-var secretKeyMaxLen = secretKeyMaxLenAmazon
-
-func mustGetAccessKey() string {
-	keyBytes := make([]byte, accessKeyMaxLen)
-	if _, err := rand.Read(keyBytes); err != nil {
-		console.Fatalf("Unable to generate access key. Err: %s.\n", err)
-	}
-
-	for i := 0; i < accessKeyMaxLen; i++ {
-		keyBytes[i] = alphaNumericTable[keyBytes[i]%alphaNumericTableLen]
-	}
-
-	return string(keyBytes)
-}
-
-func mustGetSecretKey() string {
-	keyBytes := make([]byte, secretKeyMaxLen)
-	if _, err := rand.Read(keyBytes); err != nil {
-		console.Fatalf("Unable to generate secret key. Err: %s.\n", err)
-	}
-
-	return string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLen])
-}
 
 // isAccessKeyValid - validate access key for right length.
 func isAccessKeyValid(accessKey string) bool {
-	return len(accessKey) >= accessKeyMinLen && len(accessKey) <= accessKeyMaxLen
+	return len(accessKey) >= accessKeyMinLen
 }
 
 // isSecretKeyValid - validate secret key for right length.
 func isSecretKeyValid(secretKey string) bool {
-	return len(secretKey) >= secretKeyMinLen && len(secretKey) <= secretKeyMaxLen
+	return len(secretKey) >= secretKeyMinLen
 }
 
 // credential container for access and secret keys.
@@ -124,28 +114,35 @@ func createCredential(accessKey, secretKey string) (cred credential, err error)
 }
 
 // Initialize a new credential object
-func mustGetNewCredential() credential {
+func getNewCredential(accessKeyLen, secretKeyLen int) (cred credential, err error) {
 	// Generate access key.
-	keyBytes := make([]byte, accessKeyMaxLen)
-	if _, err := rand.Read(keyBytes); err != nil {
-		console.Fatalln("Unable to generate access key.", err)
+	keyBytes := make([]byte, accessKeyLen)
+	_, err = rand.Read(keyBytes)
+	if err != nil {
+		return cred, err
 	}
-	for i := 0; i < accessKeyMaxLen; i++ {
+
+	for i := 0; i < accessKeyLen; i++ {
 		keyBytes[i] = alphaNumericTable[keyBytes[i]%alphaNumericTableLen]
 	}
 	accessKey := string(keyBytes)
 
 	// Generate secret key.
-	keyBytes = make([]byte, secretKeyMaxLen)
-	if _, err := rand.Read(keyBytes); err != nil {
-		console.Fatalln("Unable to generate secret key.", err)
-	}
-	secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLen])
-
-	cred, err := createCredential(accessKey, secretKey)
+	keyBytes = make([]byte, secretKeyLen)
+	_, err = rand.Read(keyBytes)
 	if err != nil {
-		console.Fatalln("Unable to generate new credential.", err)
+		return cred, err
 	}
 
+	secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyLen])
+	cred, err = createCredential(accessKey, secretKey)
+
+	return cred, err
+}
+
+func mustGetNewCredential() credential {
+	// Generate Minio credentials with Minio key max lengths.
+	cred, err := getNewCredential(accessKeyMaxLen, secretKeyMaxLen)
+	fatalIf(err, "Unable to generate new credentials.")
 	return cred
 }

cmd/credential_test.go

@@ -23,6 +23,9 @@ func TestMustGetNewCredential(t *testing.T) {
 	if !cred.IsValid() {
 		t.Fatalf("Failed to get new valid credential")
 	}
+	if len(cred.SecretKey) != secretKeyMaxLen {
+		t.Fatalf("Invalid length %d of the secretKey credential generated, expected %d", len(cred.SecretKey), secretKeyMaxLen)
+	}
 }
 
 func TestCreateCredential(t *testing.T) {
@@ -33,18 +36,14 @@ func TestCreateCredential(t *testing.T) {
 		expectedResult bool
 		expectedErr    error
 	}{
-		// Access key too small.
+		// Access key too small (min 5 chars).
 		{"user", "pass", false, errInvalidAccessKeyLength},
-		// Access key too long.
-		{"user12345678901234567", "pass", false, errInvalidAccessKeyLength},
-		// Access key contains unsuppported characters.
-		{"!@#$", "pass", false, errInvalidAccessKeyLength},
-		// Secret key too small.
+		// Long access key is ok.
+		{"user123456789012345678901234567890", "password", true, nil},
+		// Secret key too small (min 8 chars).
 		{"myuser", "pass", false, errInvalidSecretKeyLength},
-		// Secret key too long.
-		{"myuser", "pass1234567890123456789012345678901234567", false, errInvalidSecretKeyLength},
-		// Success when access key contains leading/trailing spaces.
-		{" user ", cred.SecretKey, true, nil},
+		// Long secret key is ok.
+		{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", true, nil},
 		{"myuser", "mypassword", true, nil},
 		{cred.AccessKey, cred.SecretKey, true, nil},
 	}

cmd/endpoint.go

@@ -0,0 +1,408 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio-go/pkg/set"
+)
+
+// EndpointType - enum for endpoint type.
+type EndpointType int
+
+const (
+	// PathEndpointType - path style endpoint type enum.
+	PathEndpointType EndpointType = iota + 1
+
+	// URLEndpointType - URL style endpoint type enum.
+	URLEndpointType
+)
+
+// Endpoint - any type of endpoint.
+type Endpoint struct {
+	*url.URL
+	IsLocal bool
+}
+
+func (endpoint Endpoint) String() string {
+	if endpoint.Host == "" {
+		return endpoint.Path
+	}
+
+	return endpoint.URL.String()
+}
+
+// Type - returns type of endpoint.
+func (endpoint Endpoint) Type() EndpointType {
+	if endpoint.Host == "" {
+		return PathEndpointType
+	}
+
+	return URLEndpointType
+}
+
+// SetHTTPS - sets secure http for URLEndpointType.
+func (endpoint Endpoint) SetHTTPS() {
+	if endpoint.Host != "" {
+		endpoint.Scheme = "https"
+	}
+}
+
+// SetHTTP - sets insecure http for URLEndpointType.
+func (endpoint Endpoint) SetHTTP() {
+	if endpoint.Host != "" {
+		endpoint.Scheme = "http"
+	}
+}
+
+// NewEndpoint - returns new endpoint based on given arguments.
+func NewEndpoint(arg string) (ep Endpoint, e error) {
+	// isEmptyPath - check whether given path is not empty.
+	isEmptyPath := func(path string) bool {
+		return path == "" || path == "/" || path == `\`
+	}
+
+	if isEmptyPath(arg) {
+		return ep, fmt.Errorf("empty or root endpoint is not supported")
+	}
+
+	var isLocal bool
+	u, err := url.Parse(arg)
+	if err == nil && u.Host != "" {
+		// URL style of endpoint.
+		// Valid URL style endpoint is
+		// - Scheme field must contain "http" or "https"
+		// - All field should be empty except Host and Path.
+		if !((u.Scheme == "http" || u.Scheme == "https") &&
+			u.User == nil && u.Opaque == "" && u.ForceQuery == false && u.RawQuery == "" && u.Fragment == "") {
+			return ep, fmt.Errorf("invalid URL endpoint format")
+		}
+
+		host, port, err := net.SplitHostPort(u.Host)
+		if err != nil {
+			if !strings.Contains(err.Error(), "missing port in address") {
+				return ep, fmt.Errorf("invalid URL endpoint format: %s", err)
+			}
+
+			host = u.Host
+		} else {
+			var p int
+			p, err = strconv.Atoi(port)
+			if err != nil {
+				return ep, fmt.Errorf("invalid URL endpoint format: invalid port number")
+			} else if p < 1 || p > 65535 {
+				return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
+			}
+		}
+
+		if host == "" {
+			return ep, fmt.Errorf("invalid URL endpoint format: empty host name")
+		}
+
+		// As this is path in the URL, we should use path package, not filepath package.
+		// On MS Windows, filepath.Clean() converts into Windows path style ie `/foo` becomes `\foo`
+		u.Path = path.Clean(u.Path)
+		if isEmptyPath(u.Path) {
+			return ep, fmt.Errorf("empty or root path is not supported in URL endpoint")
+		}
+
+		isLocal, err = isLocalHost(host)
+		if err != nil {
+			return ep, err
+		}
+	} else {
+		u = &url.URL{Path: path.Clean(arg)}
+		isLocal = true
+	}
+
+	return Endpoint{
+		URL:     u,
+		IsLocal: isLocal,
+	}, nil
+}
+
+// EndpointList - list of same type of endpoint.
+type EndpointList []Endpoint
+
+// Swap - helper method for sorting.
+func (endpoints EndpointList) Swap(i, j int) {
+	endpoints[i], endpoints[j] = endpoints[j], endpoints[i]
+}
+
+// Len - helper method for sorting.
+func (endpoints EndpointList) Len() int {
+	return len(endpoints)
+}
+
+// Less - helper method for sorting.
+func (endpoints EndpointList) Less(i, j int) bool {
+	return endpoints[i].String() < endpoints[j].String()
+}
+
+// SetHTTPS - sets secure http for URLEndpointType.
+func (endpoints EndpointList) SetHTTPS() {
+	for i := range endpoints {
+		endpoints[i].SetHTTPS()
+	}
+}
+
+// SetHTTP - sets insecure http for URLEndpointType.
+func (endpoints EndpointList) SetHTTP() {
+	for i := range endpoints {
+		endpoints[i].SetHTTP()
+	}
+}
+
+// NewEndpointList - returns new endpoint list based on input args.
+func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
+	// isValidDistribution - checks whether given count is a valid distribution for erasure coding.
+	isValidDistribution := func(count int) bool {
+		return (count >= minErasureBlocks && count <= maxErasureBlocks && count%2 == 0)
+	}
+
+	// Check whether no. of args are valid for XL distribution.
+	if !isValidDistribution(len(args)) {
+		return nil, fmt.Errorf("A total of %d endpoints were found. For erasure mode it should be an even number between %d and %d", len(args), minErasureBlocks, maxErasureBlocks)
+	}
+
+	var endpointType EndpointType
+	var scheme string
+
+	uniqueArgs := set.NewStringSet()
+	// Loop through args and adds to endpoint list.
+	for i, arg := range args {
+		endpoint, err := NewEndpoint(arg)
+		if err != nil {
+			return nil, fmt.Errorf("'%s': %s", arg, err.Error())
+		}
+
+		// All endpoints have to be same type and scheme if applicable.
+		if i == 0 {
+			endpointType = endpoint.Type()
+			scheme = endpoint.Scheme
+		} else if endpoint.Type() != endpointType {
+			return nil, fmt.Errorf("mixed style endpoints are not supported")
+		} else if endpoint.Scheme != scheme {
+			return nil, fmt.Errorf("mixed scheme is not supported")
+		}
+
+		arg = endpoint.String()
+		if uniqueArgs.Contains(arg) {
+			return nil, fmt.Errorf("duplicate endpoints found")
+		}
+		uniqueArgs.Add(arg)
+
+		endpoints = append(endpoints, endpoint)
+	}
+
+	sort.Sort(endpoints)
+
+	return endpoints, nil
+}
+
+// CreateEndpoints - validates and creates new endpoints for given args.
+func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, error) {
+	var endpoints EndpointList
+	var setupType SetupType
+	var err error
+
+	// Check whether serverAddr is valid for this host.
+	if err = CheckLocalServerAddr(serverAddr); err != nil {
+		return serverAddr, endpoints, setupType, err
+	}
+
+	_, serverAddrPort := mustSplitHostPort(serverAddr)
+
+	// For single arg, return FS setup.
+	if len(args) == 1 {
+		var endpoint Endpoint
+		endpoint, err = NewEndpoint(args[0])
+		if err != nil {
+			return serverAddr, endpoints, setupType, err
+		}
+
+		if endpoint.Type() != PathEndpointType {
+			return serverAddr, endpoints, setupType, fmt.Errorf("use path style endpoint for FS setup")
+		}
+
+		endpoints = append(endpoints, endpoint)
+		setupType = FSSetupType
+		return serverAddr, endpoints, setupType, nil
+	}
+
+	// Convert args to endpoints
+	if endpoints, err = NewEndpointList(args...); err != nil {
+		return serverAddr, endpoints, setupType, err
+	}
+
+	// Return XL setup when all endpoints are path style.
+	if endpoints[0].Type() == PathEndpointType {
+		setupType = XLSetupType
+		return serverAddr, endpoints, setupType, nil
+	}
+
+	// Here all endpoints are URL style.
+	endpointPathSet := set.NewStringSet()
+	localEndpointCount := 0
+	localServerAddrSet := set.NewStringSet()
+	localPortSet := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		endpointPathSet.Add(endpoint.Path)
+		if endpoint.IsLocal {
+			localServerAddrSet.Add(endpoint.Host)
+
+			var port string
+			_, port, err = net.SplitHostPort(endpoint.Host)
+			if err != nil {
+				port = serverAddrPort
+			}
+
+			localPortSet.Add(port)
+
+			localEndpointCount++
+		}
+	}
+
+	// No local endpoint found.
+	if localEndpointCount == 0 {
+		return serverAddr, endpoints, setupType, fmt.Errorf("no endpoint found for this host")
+	}
+
+	// Check whether same path is not used in endpoints of a host.
+	{
+		pathIPMap := make(map[string]set.StringSet)
+		for _, endpoint := range endpoints {
+			var host string
+			host, _, err = net.SplitHostPort(endpoint.Host)
+			if err != nil {
+				host = endpoint.Host
+			}
+			hostIPSet, _ := getHostIP4(host)
+			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
+				if !IPSet.Intersection(hostIPSet).IsEmpty() {
+					err = fmt.Errorf("path '%s' can not be served by different port on same address", endpoint.Path)
+					return serverAddr, endpoints, setupType, err
+				}
+
+				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
+			} else {
+				pathIPMap[endpoint.Path] = hostIPSet
+			}
+		}
+	}
+
+	// Check whether serverAddrPort matches at least in one of port used in local endpoints.
+	{
+		if !localPortSet.Contains(serverAddrPort) {
+			if len(localPortSet) > 1 {
+				err = fmt.Errorf("port number in server address must match with one of the port in local endpoints")
+			} else {
+				err = fmt.Errorf("server address and local endpoint have different ports")
+			}
+
+			return serverAddr, endpoints, setupType, err
+		}
+	}
+
+	// All endpoints are pointing to local host
+	if len(endpoints) == localEndpointCount {
+		// If all endpoints have same port number, then this is XL setup using URL style endpoints.
+		if len(localPortSet) == 1 {
+			if len(localServerAddrSet) > 1 {
+				// TODO: Eventhough all endpoints are local, the local host is referred by different IP/name.
+				// eg '172.0.0.1', 'localhost' and 'mylocalhostname' point to same local host.
+				//
+				// In this case, we bind to 0.0.0.0 ie to all interfaces.
+				// The actual way to do is bind to only IPs in uniqueLocalHosts.
+				serverAddr = net.JoinHostPort("", serverAddrPort)
+			}
+
+			endpointPaths := endpointPathSet.ToSlice()
+			endpoints, _ = NewEndpointList(endpointPaths...)
+			setupType = XLSetupType
+			return serverAddr, endpoints, setupType, nil
+		}
+
+		// Eventhough all endpoints are local, but those endpoints use different ports.
+		// This means it is DistXL setup.
+	} else {
+		// This is DistXL setup.
+		// Check whether local server address are not 127.x.x.x
+		for _, localServerAddr := range localServerAddrSet.ToSlice() {
+			host, _, err := net.SplitHostPort(localServerAddr)
+			if err != nil {
+				host = localServerAddr
+			}
+
+			ipList, err := getHostIP4(host)
+			fatalIf(err, "unexpected error when resolving host '%s'", host)
+
+			// Filter ipList by IPs those start with '127.'.
+			loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool {
+				return strings.HasPrefix(ip, "127.")
+			}, "")
+
+			// If loop back IP is found and ipList contains only loop back IPs, then error out.
+			if len(loopBackIPs) > 0 && len(loopBackIPs) == len(ipList) {
+				err = fmt.Errorf("'%s' resolves to loopback address is not allowed for distributed XL", localServerAddr)
+				return serverAddr, endpoints, setupType, err
+			}
+		}
+	}
+
+	// Add missing port in all endpoints.
+	for i := range endpoints {
+		_, port, err := net.SplitHostPort(endpoints[i].Host)
+		if err != nil {
+			endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
+		} else if endpoints[i].IsLocal && serverAddrPort != port {
+			// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
+			endpoints[i].IsLocal = false
+		}
+	}
+
+	setupType = DistXLSetupType
+	return serverAddr, endpoints, setupType, nil
+}
+
+// GetRemotePeers - get hosts information other than this minio service.
+func GetRemotePeers(endpoints EndpointList) []string {
+	peerSet := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if endpoint.Type() != URLEndpointType {
+			continue
+		}
+
+		peer := endpoint.Host
+		if endpoint.IsLocal {
+			if _, port := mustSplitHostPort(peer); port == globalMinioPort {
+				continue
+			}
+		}
+
+		peerSet.Add(peer)
+	}
+
+	return peerSet.ToSlice()
+}

cmd/endpoint_test.go

@@ -0,0 +1,371 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"net/url"
+	"reflect"
+	"runtime"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func TestNewEndpoint(t *testing.T) {
+	u1, _ := url.Parse("http://localhost/path")
+	u2, _ := url.Parse("https://example.org/path")
+	u3, _ := url.Parse("http://127.0.0.1:8080/path")
+	u4, _ := url.Parse("http://192.168.253.200/path")
+
+	errMsg := ": no such host"
+	if runtime.GOOS == "windows" {
+		errMsg = ": No such host is known."
+	}
+
+	testCases := []struct {
+		arg              string
+		expectedEndpoint Endpoint
+		expectedType     EndpointType
+		expectedErr      error
+	}{
+		{"foo", Endpoint{URL: &url.URL{Path: "foo"}, IsLocal: true}, PathEndpointType, nil},
+		{"/foo", Endpoint{URL: &url.URL{Path: "/foo"}, IsLocal: true}, PathEndpointType, nil},
+		{`\foo`, Endpoint{URL: &url.URL{Path: `\foo`}, IsLocal: true}, PathEndpointType, nil},
+		{"C", Endpoint{URL: &url.URL{Path: `C`}, IsLocal: true}, PathEndpointType, nil},
+		{"C:", Endpoint{URL: &url.URL{Path: `C:`}, IsLocal: true}, PathEndpointType, nil},
+		{"C:/", Endpoint{URL: &url.URL{Path: "C:"}, IsLocal: true}, PathEndpointType, nil},
+		{`C:\`, Endpoint{URL: &url.URL{Path: `C:\`}, IsLocal: true}, PathEndpointType, nil},
+		{`C:\foo`, Endpoint{URL: &url.URL{Path: `C:\foo`}, IsLocal: true}, PathEndpointType, nil},
+		{"C:/foo", Endpoint{URL: &url.URL{Path: "C:/foo"}, IsLocal: true}, PathEndpointType, nil},
+		{`C:\\foo`, Endpoint{URL: &url.URL{Path: `C:\\foo`}, IsLocal: true}, PathEndpointType, nil},
+		{"http:path", Endpoint{URL: &url.URL{Path: "http:path"}, IsLocal: true}, PathEndpointType, nil},
+		{"http:/path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
+		{"http:///path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
+		{"http://localhost/path", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
+		{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
+		{"https://example.org/path", Endpoint{URL: u2}, URLEndpointType, nil},
+		{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil},
+		{"http://192.168.253.200/path", Endpoint{URL: u4}, URLEndpointType, nil},
+		{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
+		{"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
+		{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
+		{"c://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
+		{"ftp://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
+		{"http://server/path?location", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
+		{"http://:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")},
+		{"http://:8080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: empty host name")},
+		{"http://server:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")},
+		{"https://93.184.216.34:808080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")},
+		{"http://server:8080//", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
+		{"http://server:8080/", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
+		{"http://server/path", Endpoint{}, -1, fmt.Errorf("lookup server" + errMsg)},
+	}
+
+	for _, testCase := range testCases {
+		endpoint, err := NewEndpoint(testCase.arg)
+		if testCase.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+		} else {
+			var match bool
+			if strings.HasSuffix(testCase.expectedErr.Error(), errMsg) {
+				match = strings.HasSuffix(err.Error(), errMsg)
+			} else {
+				match = (testCase.expectedErr.Error() == err.Error())
+			}
+			if !match {
+				t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+			}
+		}
+
+		if err == nil && !reflect.DeepEqual(testCase.expectedEndpoint, endpoint) {
+			t.Fatalf("endpoint: expected = %+v, got = %+v", testCase.expectedEndpoint, endpoint)
+		}
+
+		if err == nil && testCase.expectedType != endpoint.Type() {
+			t.Fatalf("type: expected = %+v, got = %+v", testCase.expectedType, endpoint.Type())
+		}
+	}
+}
+
+func TestNewEndpointList(t *testing.T) {
+	testCases := []struct {
+		args        []string
+		expectedErr error
+	}{
+		{[]string{"d1", "d2", "d3", "d4"}, nil},
+		{[]string{"/d1", "/d2", "/d3", "/d4"}, nil},
+		{[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, nil},
+		{[]string{"http://example.org/d1", "http://example.com/d1", "http://example.net/d1", "http://example.edu/d1"}, nil},
+		{[]string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d1", "http://example.org/d2"}, nil},
+		{[]string{"https://localhost:9000/d1", "https://localhost:9001/d2", "https://localhost:9002/d3", "https://localhost:9003/d4"}, nil},
+		// // It is valid WRT endpoint list that same path is expected with different port on same server.
+		{[]string{"https://127.0.0.1:9000/d1", "https://127.0.0.1:9001/d1", "https://127.0.0.1:9002/d1", "https://127.0.0.1:9003/d1"}, nil},
+		{[]string{"d1", "d2", "d3", "d1"}, fmt.Errorf("duplicate endpoints found")},
+		{[]string{"d1", "d2", "d3", "./d1"}, fmt.Errorf("duplicate endpoints found")},
+		{[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d1", "http://localhost/d4"}, fmt.Errorf("duplicate endpoints found")},
+		{[]string{"d1", "d2", "d3", "d4", "d5"}, fmt.Errorf("A total of 5 endpoints were found. For erasure mode it should be an even number between 4 and 16")},
+		{[]string{"ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"}, fmt.Errorf("'ftp://server/d1': invalid URL endpoint format")},
+		{[]string{"d1", "http://localhost/d2", "d3", "d4"}, fmt.Errorf("mixed style endpoints are not supported")},
+		{[]string{"http://example.org/d1", "https://example.com/d1", "http://example.net/d1", "https://example.edut/d1"}, fmt.Errorf("mixed scheme is not supported")},
+	}
+
+	for _, testCase := range testCases {
+		_, err := NewEndpointList(testCase.args...)
+		if testCase.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+		} else if testCase.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+	}
+}
+
+func TestCreateEndpoints(t *testing.T) {
+	// Filter ipList by IPs those do not start with '127.'.
+	nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool {
+		return !strings.HasPrefix(ip, "127.")
+	}, "")
+	if len(nonLoopBackIPs) == 0 {
+		t.Fatalf("No non-loop back IP address found for this host")
+	}
+	nonLoopBackIP := nonLoopBackIPs.ToSlice()[0]
+
+	getExpectedEndpoints := func(args []string, prefix string) ([]*url.URL, []bool) {
+		var URLs []*url.URL
+		var localFlags []bool
+		sort.Strings(args)
+		for _, arg := range args {
+			u, _ := url.Parse(arg)
+			URLs = append(URLs, u)
+			localFlags = append(localFlags, strings.HasPrefix(arg, prefix))
+		}
+
+		return URLs, localFlags
+	}
+
+	case1Endpoint1 := "http://" + nonLoopBackIP + "/d1"
+	case1Endpoint2 := "http://" + nonLoopBackIP + "/d2"
+	args := []string{
+		"http://" + nonLoopBackIP + ":10000/d1",
+		"http://" + nonLoopBackIP + ":10000/d2",
+		"http://example.com:10000/d4",
+		"http://example.org:10000/d3",
+	}
+	case1URLs, case1LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":10000/")
+
+	case2Endpoint1 := "http://" + nonLoopBackIP + "/d1"
+	case2Endpoint2 := "http://" + nonLoopBackIP + ":9000/d2"
+	args = []string{
+		"http://" + nonLoopBackIP + ":10000/d1",
+		"http://" + nonLoopBackIP + ":9000/d2",
+		"http://example.com:10000/d4",
+		"http://example.org:10000/d3",
+	}
+	case2URLs, case2LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":10000/")
+
+	case3Endpoint1 := "http://" + nonLoopBackIP + "/d1"
+	args = []string{
+		"http://" + nonLoopBackIP + ":80/d1",
+		"http://example.com:80/d3",
+		"http://example.net:80/d4",
+		"http://example.org:9000/d2",
+	}
+	case3URLs, case3LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":80/")
+
+	case4Endpoint1 := "http://" + nonLoopBackIP + "/d1"
+	args = []string{
+		"http://" + nonLoopBackIP + ":9000/d1",
+		"http://example.com:9000/d3",
+		"http://example.net:9000/d4",
+		"http://example.org:9000/d2",
+	}
+	case4URLs, case4LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9000/")
+
+	case5Endpoint1 := "http://" + nonLoopBackIP + ":9000/d1"
+	case5Endpoint2 := "http://" + nonLoopBackIP + ":9001/d2"
+	case5Endpoint3 := "http://" + nonLoopBackIP + ":9002/d3"
+	case5Endpoint4 := "http://" + nonLoopBackIP + ":9003/d4"
+	args = []string{
+		case5Endpoint1,
+		case5Endpoint2,
+		case5Endpoint3,
+		case5Endpoint4,
+	}
+	case5URLs, case5LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9000/")
+
+	case6Endpoint := "http://" + nonLoopBackIP + ":9003/d4"
+	args = []string{
+		"http://localhost:9000/d1",
+		"http://localhost:9001/d2",
+		"http://127.0.0.1:9002/d3",
+		case6Endpoint,
+	}
+	case6URLs, case6LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9003/")
+
+	testCases := []struct {
+		serverAddr         string
+		args               []string
+		expectedServerAddr string
+		expectedEndpoints  EndpointList
+		expectedSetupType  SetupType
+		expectedErr        error
+	}{
+		{"localhost", []string{}, "", EndpointList{}, -1, fmt.Errorf("missing port in address localhost")},
+
+		// FS Setup
+		{"localhost:9000", []string{"http://localhost/d1"}, "", EndpointList{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
+		{":443", []string{"d1"}, ":443", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
+		{"localhost:10000", []string{"/d1"}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true}}, FSSetupType, nil},
+		{"localhost:10000", []string{"./d1"}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
+		{"localhost:10000", []string{`\d1`}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `\d1`}, IsLocal: true}}, FSSetupType, nil},
+		{"localhost:10000", []string{`.\d1`}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `.\d1`}, IsLocal: true}}, FSSetupType, nil},
+		{":8080", []string{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
+		{":8080", []string{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
+		{"localhost:9000", []string{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
+		{"localhost:9000", []string{"https://127.0.0.1:8000/d1", "https://localhost:9001/d2", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("port number in server address must match with one of the port in local endpoints")},
+		{"localhost:10000", []string{"https://127.0.0.1:8000/d1", "https://localhost:8000/d2", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("server address and local endpoint have different ports")},
+
+		// XL Setup with PathEndpointType
+		{":1234", []string{"/d1", "/d2", "d3", "d4"}, ":1234",
+			EndpointList{
+				Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
+				Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
+				Endpoint{URL: &url.URL{Path: "d3"}, IsLocal: true},
+				Endpoint{URL: &url.URL{Path: "d4"}, IsLocal: true},
+			}, XLSetupType, nil},
+		// XL Setup with URLEndpointType
+		{":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, ":9000", EndpointList{
+			Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
+		}, XLSetupType, nil},
+		// XL Setup with URLEndpointType having mixed naming to local host.
+		{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}, ":10000", EndpointList{
+			Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
+		}, XLSetupType, nil},
+		{":9001", []string{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}, "", EndpointList{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
+
+		{":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d3", "http://example.com/d4"}, "", EndpointList{}, -1, fmt.Errorf("'localhost' resolves to loopback address is not allowed for distributed XL")},
+
+		// DistXL type
+		{"127.0.0.1:10000", []string{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", EndpointList{
+			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
+			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
+			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
+			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
+		}, DistXLSetupType, nil},
+
+		{"127.0.0.1:10000", []string{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", EndpointList{
+			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
+			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
+			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
+			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
+		}, DistXLSetupType, nil},
+
+		{":80", []string{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}, ":80", EndpointList{
+			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
+			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
+			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
+			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
+		}, DistXLSetupType, nil},
+
+		{":9000", []string{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}, ":9000", EndpointList{
+			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
+			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
+			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
+			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
+		}, DistXLSetupType, nil},
+
+		{":9000", []string{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}, ":9000", EndpointList{
+			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
+			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
+			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
+			Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]},
+		}, DistXLSetupType, nil},
+
+		// DistXL Setup using only local host.
+		{":9003", []string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}, ":9003", EndpointList{
+			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
+			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
+			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
+			Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]},
+		}, DistXLSetupType, nil},
+	}
+
+	for _, testCase := range testCases {
+		serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
+
+		if err == nil {
+			if testCase.expectedErr != nil {
+				t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+			} else {
+				if serverAddr != testCase.expectedServerAddr {
+					t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
+				}
+				if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
+					t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
+				}
+				if setupType != testCase.expectedSetupType {
+					t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
+				}
+			}
+		} else if testCase.expectedErr == nil {
+			t.Fatalf("error: expected = <nil>, got = %v", err)
+		} else if err.Error() != testCase.expectedErr.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+	}
+}
+
+func TestGetRemotePeers(t *testing.T) {
+	tempGlobalMinioPort := globalMinioPort
+	defer func() {
+		globalMinioPort = tempGlobalMinioPort
+	}()
+	globalMinioPort = "9000"
+
+	testCases := []struct {
+		endpointArgs   []string
+		expectedResult []string
+	}{
+		{[]string{"/d1", "/d2", "d3", "d4"}, []string{}},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000"}},
+		{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000"}},
+		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000"}},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9001", "localhost:9002", "localhost:9003"}},
+	}
+
+	for _, testCase := range testCases {
+		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
+		remotePeers := GetRemotePeers(endpoints)
+		if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
+			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
+		}
+	}
+}

cmd/erasure-createfile.go

@@ -28,7 +28,9 @@ import (
 // erasureCreateFile - writes an entire stream by erasure coding to
 // all the disks, writes also calculate individual block's checksum
 // for future bit-rot protection.
-func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, allowEmpty bool, blockSize int64, dataBlocks int, parityBlocks int, algo string, writeQuorum int) (bytesWritten int64, checkSums []string, err error) {
+func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, allowEmpty bool, blockSize int64,
+	dataBlocks, parityBlocks int, algo HashAlgo, writeQuorum int) (newDisks []StorageAPI, bytesWritten int64, checkSums []string, err error) {
+
 	// Allocated blockSized buffer for reading from incoming stream.
 	buf := make([]byte, blockSize)
 
@@ -41,7 +43,7 @@ func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader
 		// FIXME: this is a bug in Golang, n == 0 and err ==
 		// io.ErrUnexpectedEOF for io.ReadFull function.
 		if n == 0 && rErr == io.ErrUnexpectedEOF {
-			return 0, nil, traceError(rErr)
+			return nil, 0, nil, traceError(rErr)
 		}
 		if rErr == io.EOF {
 			// We have reached EOF on the first byte read, io.Reader
@@ -49,28 +51,28 @@ func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader
 			// data. Will create a 0byte file instead.
 			if bytesWritten == 0 && allowEmpty {
 				blocks = make([][]byte, len(disks))
-				rErr = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum)
+				newDisks, rErr = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum)
 				if rErr != nil {
-					return 0, nil, rErr
+					return nil, 0, nil, rErr
 				}
 			} // else we have reached EOF after few reads, no need to
 			// add an additional 0bytes at the end.
 			break
 		}
 		if rErr != nil && rErr != io.ErrUnexpectedEOF {
-			return 0, nil, traceError(rErr)
+			return nil, 0, nil, traceError(rErr)
 		}
 		if n > 0 {
 			// Returns encoded blocks.
 			var enErr error
 			blocks, enErr = encodeData(buf[0:n], dataBlocks, parityBlocks)
 			if enErr != nil {
-				return 0, nil, enErr
+				return nil, 0, nil, enErr
 			}
 
 			// Write to all disks.
-			if err = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum); err != nil {
-				return 0, nil, err
+			if newDisks, err = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum); err != nil {
+				return nil, 0, nil, err
 			}
 			bytesWritten += int64(n)
 		}
@@ -80,7 +82,7 @@ func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader
 	for i := range checkSums {
 		checkSums[i] = hex.EncodeToString(hashWriters[i].Sum(nil))
 	}
-	return bytesWritten, checkSums, nil
+	return newDisks, bytesWritten, checkSums, nil
 }
 
 // encodeData - encodes incoming data buffer into
@@ -108,7 +110,7 @@ func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, erro
 }
 
 // appendFile - append data buffer at path.
-func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hashWriters []hash.Hash, writeQuorum int) (err error) {
+func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hashWriters []hash.Hash, writeQuorum int) ([]StorageAPI, error) {
 	var wg = &sync.WaitGroup{}
 	var wErrs = make([]error, len(disks))
 	// Write encoded data to quorum disks in parallel.
@@ -124,8 +126,6 @@ func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hash
 			wErr := disk.AppendFile(volume, path, enBlocks[index])
 			if wErr != nil {
 				wErrs[index] = traceError(wErr)
-				// Ignore disk which returned an error.
-				disks[index] = nil
 				return
 			}
 
@@ -140,5 +140,5 @@ func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hash
 	// Wait for all the appends to finish.
 	wg.Wait()
 
-	return reduceWriteQuorumErrs(wErrs, objectOpIgnoredErrs, writeQuorum)
+	return evalDisks(disks, wErrs), reduceWriteQuorumErrs(wErrs, objectOpIgnoredErrs, writeQuorum)
 }

cmd/erasure-createfile_test.go

@@ -56,7 +56,7 @@ func TestErasureCreateFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Test when all disks are up.
-	size, _, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, _, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -69,7 +69,7 @@ func TestErasureCreateFile(t *testing.T) {
 	disks[5] = AppendDiskDown{disks[5].(*posix)}
 
 	// Test when two disks are down.
-	size, _, err = erasureCreateFile(disks, "testbucket", "testobject2", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, _, err = erasureCreateFile(disks, "testbucket", "testobject2", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -83,7 +83,7 @@ func TestErasureCreateFile(t *testing.T) {
 	disks[8] = AppendDiskDown{disks[8].(*posix)}
 	disks[9] = AppendDiskDown{disks[9].(*posix)}
 
-	size, _, err = erasureCreateFile(disks, "testbucket", "testobject3", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, _, err = erasureCreateFile(disks, "testbucket", "testobject3", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -93,7 +93,7 @@ func TestErasureCreateFile(t *testing.T) {
 
 	// 1 more disk down. 7 disk down in total. Should return quorum error.
 	disks[10] = AppendDiskDown{disks[10].(*posix)}
-	_, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, _, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if errorCause(err) != errXLWriteQuorum {
 		t.Errorf("erasureCreateFile return value: expected errXLWriteQuorum, got %s", err)
 	}

cmd/erasure-healfile.go

@@ -19,7 +19,9 @@ package cmd
 import "encoding/hex"
 
 // Heals the erasure coded file. reedsolomon.Reconstruct() is used to reconstruct the missing parts.
-func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string, size int64, blockSize int64, dataBlocks int, parityBlocks int, algo string) (checkSums []string, err error) {
+func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string,
+	size, blockSize int64, dataBlocks, parityBlocks int, algo HashAlgo) (checkSums []string, err error) {
+
 	var offset int64
 	remainingSize := size
 

cmd/erasure-healfile_test.go

@@ -48,7 +48,7 @@ func TestErasureHealFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Create a test file.
-	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/erasure-readfile.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"encoding/hex"
 	"errors"
 	"io"
 	"sync"
@@ -111,7 +110,9 @@ func getReadDisks(orderedDisks []StorageAPI, index int, dataBlocks int) (readDis
 }
 
 // parallelRead - reads chunks in parallel from the disks specified in []readDisks.
-func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []StorageAPI, enBlocks [][]byte, blockOffset int64, curChunkSize int64, bitRotVerify func(diskIndex int) bool, pool *bpool.BytePool) {
+func parallelRead(volume, path string, readDisks, orderedDisks []StorageAPI, enBlocks [][]byte,
+	blockOffset, curChunkSize int64, brVerifiers []bitRotVerifier, pool *bpool.BytePool) {
+
 	// WaitGroup to synchronise the read go-routines.
 	wg := &sync.WaitGroup{}
 
@@ -125,11 +126,15 @@ func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []St
 		go func(index int) {
 			defer wg.Done()
 
-			// Verify bit rot for the file on this disk.
-			if !bitRotVerify(index) {
-				// So that we don't read from this disk for the next block.
-				orderedDisks[index] = nil
-				return
+			// evaluate if we need to perform bit-rot checking
+			needBitRotVerification := true
+			if brVerifiers[index].isVerified {
+				needBitRotVerification = false
+				// if file has bit-rot, do not reuse disk
+				if brVerifiers[index].hasBitRot {
+					orderedDisks[index] = nil
+					return
+				}
 			}
 
 			buf, err := pool.Get()
@@ -140,7 +145,25 @@ func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []St
 			}
 			buf = buf[:curChunkSize]
 
-			_, err = readDisks[index].ReadFile(volume, path, blockOffset, buf)
+			if needBitRotVerification {
+				_, err = readDisks[index].ReadFileWithVerify(
+					volume, path, blockOffset, buf,
+					brVerifiers[index].algo,
+					brVerifiers[index].checkSum)
+			} else {
+				_, err = readDisks[index].ReadFile(volume, path,
+					blockOffset, buf)
+			}
+
+			// if bit-rot verification was done, store the
+			// result of verification so we can skip
+			// re-doing it next time
+			if needBitRotVerification {
+				brVerifiers[index].isVerified = true
+				_, ok := err.(hashMismatchError)
+				brVerifiers[index].hasBitRot = ok
+			}
+
 			if err != nil {
 				orderedDisks[index] = nil
 				return
@@ -153,12 +176,16 @@ func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []St
 	wg.Wait()
 }
 
-// erasureReadFile - read bytes from erasure coded files and writes to given writer.
-// Erasure coded files are read block by block as per given erasureInfo and data chunks
-// are decoded into a data block. Data block is trimmed for given offset and length,
-// then written to given writer. This function also supports bit-rot detection by
-// verifying checksum of individual block's checksum.
-func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path string, offset int64, length int64, totalLength int64, blockSize int64, dataBlocks int, parityBlocks int, checkSums []string, algo string, pool *bpool.BytePool) (int64, error) {
+// erasureReadFile - read bytes from erasure coded files and writes to
+// given writer.  Erasure coded files are read block by block as per
+// given erasureInfo and data chunks are decoded into a data
+// block. Data block is trimmed for given offset and length, then
+// written to given writer. This function also supports bit-rot
+// detection by verifying checksum of individual block's checksum.
+func erasureReadFile(writer io.Writer, disks []StorageAPI, volume, path string,
+	offset, length, totalLength, blockSize int64, dataBlocks, parityBlocks int,
+	checkSums []string, algo HashAlgo, pool *bpool.BytePool) (int64, error) {
+
 	// Offset and length cannot be negative.
 	if offset < 0 || length < 0 {
 		return 0, traceError(errUnexpected)
@@ -169,27 +196,15 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s
 		return 0, traceError(errUnexpected)
 	}
 
-	// chunkSize is the amount of data that needs to be read from each disk at a time.
+	// chunkSize is the amount of data that needs to be read from
+	// each disk at a time.
 	chunkSize := getChunkSize(blockSize, dataBlocks)
 
-	// bitRotVerify verifies if the file on a particular disk doesn't have bitrot
-	// by verifying the hash of the contents of the file.
-	bitRotVerify := func() func(diskIndex int) bool {
-		verified := make([]bool, len(disks))
-		// Return closure so that we have reference to []verified and
-		// not recalculate the hash on it every time the function is
-		// called for the same disk.
-		return func(diskIndex int) bool {
-			if verified[diskIndex] {
-				// Already validated.
-				return true
-			}
-			// Is this a valid block?
-			isValid := isValidBlock(disks[diskIndex], volume, path, checkSums[diskIndex], algo)
-			verified[diskIndex] = isValid
-			return isValid
-		}
-	}()
+	brVerifiers := make([]bitRotVerifier, len(disks))
+	for i := range brVerifiers {
+		brVerifiers[i].algo = algo
+		brVerifiers[i].checkSum = checkSums[i]
+	}
 
 	// Total bytes written to writer
 	var bytesWritten int64
@@ -241,7 +256,7 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s
 				return bytesWritten, err
 			}
 			// Issue a parallel read across the disks specified in readDisks.
-			parallelRead(volume, path, readDisks, disks, enBlocks, blockOffset, curChunkSize, bitRotVerify, pool)
+			parallelRead(volume, path, readDisks, disks, enBlocks, blockOffset, curChunkSize, brVerifiers, pool)
 			if isSuccessDecodeBlocks(enBlocks, dataBlocks) {
 				// If enough blocks are available to do rs.Reconstruct()
 				break
@@ -299,27 +314,6 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s
 	return bytesWritten, nil
 }
 
-// isValidBlock - calculates the checksum hash for the block and
-// validates if its correct returns true for valid cases, false otherwise.
-func isValidBlock(disk StorageAPI, volume, path, checkSum, checkSumAlgo string) (ok bool) {
-	// Disk is not available, not a valid block.
-	if disk == nil {
-		return false
-	}
-	// Checksum not available, not a valid block.
-	if checkSum == "" {
-		return false
-	}
-	// Read everything for a given block and calculate hash.
-	hashWriter := newHash(checkSumAlgo)
-	hashBytes, err := hashSum(disk, volume, path, hashWriter)
-	if err != nil {
-		errorIf(err, "Unable to calculate checksum %s/%s", volume, path)
-		return false
-	}
-	return hex.EncodeToString(hashBytes) == checkSum
-}
-
 // decodeData - decode encoded blocks.
 func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
 	// Initialized reedsolomon.

cmd/erasure-readfile_test.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"math/rand"
 	"testing"
-	"time"
 
 	"reflect"
 
@@ -195,11 +194,7 @@ func TestErasureReadUtils(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	endpoints, err := parseStorageEndpoints(disks)
-	if err != nil {
-		t.Fatal(err)
-	}
-	objLayer, _, err := initObjectLayer(endpoints)
+	objLayer, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
 	if err != nil {
 		removeRoots(disks)
 		t.Fatal(err)
@@ -218,6 +213,12 @@ func (r ReadDiskDown) ReadFile(volume string, path string, offset int64, buf []b
 	return 0, errFaultyDisk
 }
 
+func (r ReadDiskDown) ReadFileWithVerify(volume string, path string, offset int64, buf []byte,
+	algo HashAlgo, expectedHash string) (n int64, err error) {
+
+	return 0, errFaultyDisk
+}
+
 func TestErasureReadFileDiskFail(t *testing.T) {
 	// Initialize environment needed for the test.
 	dataBlocks := 7
@@ -241,7 +242,7 @@ func TestErasureReadFileDiskFail(t *testing.T) {
 	}
 
 	// Create a test file to read from.
-	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -324,7 +325,7 @@ func TestErasureReadFileOffsetLength(t *testing.T) {
 	}
 
 	// Create a test file to read from.
-	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -403,7 +404,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	iterations := 10000
 
 	// Create a test file to read from.
-	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -412,7 +413,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	}
 
 	// To generate random offset/length.
-	r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
+	r := rand.New(rand.NewSource(UTCNow().UnixNano()))
 
 	// create pool buffer which will be used by erasureReadFile for
 	// reading from disks and erasure decoding.

cmd/erasure-utils.go

@@ -29,7 +29,7 @@ import (
 )
 
 // newHashWriters - inititialize a slice of hashes for the disk count.
-func newHashWriters(diskCount int, algo string) []hash.Hash {
+func newHashWriters(diskCount int, algo HashAlgo) []hash.Hash {
 	hashWriters := make([]hash.Hash, diskCount)
 	for index := range hashWriters {
 		hashWriters[index] = newHash(algo)
@@ -38,13 +38,13 @@ func newHashWriters(diskCount int, algo string) []hash.Hash {
 }
 
 // newHash - gives you a newly allocated hash depending on the input algorithm.
-func newHash(algo string) (h hash.Hash) {
+func newHash(algo HashAlgo) (h hash.Hash) {
 	switch algo {
-	case sha256Algo:
+	case HashSha256:
 		// sha256 checksum specially on ARM64 platforms or whenever
 		// requested as dictated by `xl.json` entry.
 		h = sha256.New()
-	case blake2bAlgo:
+	case HashBlake2b:
 		// ignore the error, because New512 without a key never fails
 		// New512 only returns a non-nil error, if the length of the passed
 		// key > 64 bytes - but we use blake2b as hash function (no key)
@@ -71,7 +71,7 @@ var hashBufferPool = sync.Pool{
 
 // hashSum calculates the hash of the entire path and returns.
 func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, error) {
-	// Fetch staging a new staging buffer from the pool.
+	// Fetch a new staging buffer from the pool.
 	bufp := hashBufferPool.Get().(*[]byte)
 	defer hashBufferPool.Put(bufp)
 
@@ -207,3 +207,16 @@ func copyBuffer(writer io.Writer, disk StorageAPI, volume string, path string, b
 	// Success.
 	return nil
 }
+
+// bitRotVerifier - type representing bit-rot verification process for
+// a single under-lying object (currently whole files)
+type bitRotVerifier struct {
+	// has the bit-rot verification been done?
+	isVerified bool
+	// is the data free of bit-rot?
+	hasBitRot bool
+	// hashing algorithm
+	algo HashAlgo
+	// hex-encoded expected raw-hash value
+	checkSum string
+}

cmd/event-notifier.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,11 +25,14 @@ import (
 	"net/url"
 	"path"
 	"sync"
-	"time"
 
 	"github.com/Sirupsen/logrus"
 )
 
+const (
+	minioEventSource = "minio:s3"
+)
+
 type externalNotifier struct {
 	// Per-bucket notification config. This is updated via
 	// PutBucketNotification API.
@@ -83,11 +86,29 @@ type eventData struct {
 	Bucket    string
 	ObjInfo   ObjectInfo
 	ReqParams map[string]string
+	Host      string
+	Port      string
+	UserAgent string
 }
 
 // New notification event constructs a new notification event message from
 // input request metadata which completed successfully.
 func newNotificationEvent(event eventData) NotificationEvent {
+	getResponseOriginEndpointKey := func() string {
+		host := globalMinioHost
+		if host == "" {
+			// FIXME: Send FQDN or hostname of this machine than sending IP address.
+			host = localIP4.ToSlice()[0]
+		}
+
+		scheme := httpScheme
+		if globalIsSSL {
+			scheme = httpsScheme
+		}
+
+		return fmt.Sprintf("%s://%s:%s", scheme, host, globalMinioPort)
+	}
+
 	// Fetch the region.
 	region := serverConfig.GetRegion()
 
@@ -95,15 +116,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
 	creds := serverConfig.GetCredential()
 
 	// Time when Minio finished processing the request.
-	eventTime := time.Now().UTC()
-
-	// API endpoint is captured here to be returned back
-	// to the client for it to differentiate from which
-	// server the request came from.
-	var apiEndpoint string
-	if len(globalAPIEndpoints) >= 1 {
-		apiEndpoint = globalAPIEndpoints[0]
-	}
+	eventTime := UTCNow()
 
 	// Fetch a hexadecimal representation of event time in nano seconds.
 	uniqueID := mustGetRequestID(eventTime)
@@ -115,7 +128,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
 	// http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
 	nEvent := NotificationEvent{
 		EventVersion:      eventVersion,
-		EventSource:       eventSource,
+		EventSource:       minioEventSource,
 		AwsRegion:         region,
 		EventTime:         eventTime.Format(timeFormatAMZ),
 		EventName:         event.Type.String(),
@@ -125,7 +138,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
 			responseRequestIDKey: uniqueID,
 			// Following is a custom response element to indicate
 			// event origin server endpoint.
-			responseOriginEndpointKey: apiEndpoint,
+			responseOriginEndpointKey: getResponseOriginEndpointKey(),
 		},
 		S3: eventMeta{
 			SchemaVersion:   eventSchemaVersion,
@@ -136,6 +149,11 @@ func newNotificationEvent(event eventData) NotificationEvent {
 				ARN:           bucketARNPrefix + event.Bucket,
 			},
 		},
+		Source: sourceInfo{
+			Host:      event.Host,
+			Port:      event.Port,
+			UserAgent: event.UserAgent,
+		},
 	}
 
 	// Escape the object name. For example "red flower.jpg" becomes "red+flower.jpg".
@@ -145,6 +163,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
 	if event.Type == ObjectRemovedDelete {
 		nEvent.S3.Object = objectMeta{
 			Key:       escapedObj,
+			VersionID: "1",
 			Sequencer: uniqueID,
 		}
 		return nEvent
@@ -152,10 +171,13 @@ func newNotificationEvent(event eventData) NotificationEvent {
 
 	// For all other events we should set ETag and Size.
 	nEvent.S3.Object = objectMeta{
-		Key:       escapedObj,
-		ETag:      event.ObjInfo.MD5Sum,
-		Size:      event.ObjInfo.Size,
-		Sequencer: uniqueID,
+		Key:         escapedObj,
+		ETag:        event.ObjInfo.ETag,
+		Size:        event.ObjInfo.Size,
+		ContentType: event.ObjInfo.ContentType,
+		UserDefined: event.ObjInfo.UserDefined,
+		VersionID:   "1",
+		Sequencer:   uniqueID,
 	}
 
 	// Success.
@@ -481,9 +503,8 @@ func removeNotificationConfig(bucket string, objAPI ObjectLayer) error {
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
 	objLock.Lock()
-	err := objAPI.DeleteObject(minioMetaBucket, ncPath)
-	objLock.Unlock()
-	return err
+	defer objLock.Unlock()
+	return objAPI.DeleteObject(minioMetaBucket, ncPath)
 }
 
 // Remove listener configuration from storage layer. Used when a bucket is deleted.
@@ -494,9 +515,8 @@ func removeListenerConfig(bucket string, objAPI ObjectLayer) error {
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
 	objLock.Lock()
-	err := objAPI.DeleteObject(minioMetaBucket, lcPath)
-	objLock.Unlock()
-	return err
+	defer objLock.Unlock()
+	return objAPI.DeleteObject(minioMetaBucket, lcPath)
 }
 
 // Loads both notification and listener config.
@@ -587,6 +607,25 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 		}
 	}
 
+	// Load all mqtt targets, initialize their respective loggers.
+	for accountID, mqttN := range serverConfig.Notify.GetMQTT() {
+		if !mqttN.Enable {
+			continue
+		}
+
+		if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeMQTT, newMQTTNotify); err != nil {
+			if _, ok := err.(net.Error); ok {
+				err = &net.OpError{
+					Op:  "Connecting to " + queueARN,
+					Net: "tcp",
+					Err: err,
+				}
+			}
+
+			return nil, err
+		}
+	}
+
 	// Load all nats targets, initialize their respective loggers.
 	for accountID, natsN := range serverConfig.Notify.GetNATS() {
 		if !natsN.Enable {
@@ -630,7 +669,6 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 		if !webhookN.Enable {
 			continue
 		}
-
 		if _, err := addQueueTarget(queueTargets, accountID, queueTypeWebhook, newWebhookNotify); err != nil {
 			return nil, err
 		}
@@ -674,6 +712,25 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 		}
 	}
 
+	// Load MySQL targets, initialize their respective loggers.
+	for accountID, msqlN := range serverConfig.Notify.GetMySQL() {
+		if !msqlN.Enable {
+			continue
+		}
+
+		if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeMySQL, newMySQLNotify); err != nil {
+			if _, ok := err.(net.Error); ok {
+				err = &net.OpError{
+					Op:  "Connecting to " + queueARN,
+					Net: "tcp",
+					Err: err,
+				}
+			}
+
+			return nil, err
+		}
+	}
+
 	// Load Kafka targets, initialize their respective loggers.
 	for accountID, kafkaN := range serverConfig.Notify.GetKafka() {
 		if !kafkaN.Enable {

cmd/event-notifier_test.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"fmt"
-	"net"
 	"reflect"
 	"testing"
 	"time"
@@ -40,17 +39,13 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
 	defer removeRoots(disks)
-	endpoints, err := parseStorageEndpoints(disks)
-	if err != nil {
-		t.Fatal(err)
-	}
-	obj, _, err := initObjectLayer(endpoints)
+	obj, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
 	if err != nil {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
 	bucketName := "bucket"
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected error:", err)
 	}
 
@@ -97,11 +92,7 @@ func TestInitEventNotifierWithPostgreSQL(t *testing.T) {
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
-	endpoints, err := parseStorageEndpoints(disks)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fs, _, err := initObjectLayer(endpoints)
+	fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
 	if err != nil {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
@@ -128,11 +119,7 @@ func TestInitEventNotifierWithNATS(t *testing.T) {
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
-	endpoints, err := parseStorageEndpoints(disks)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fs, _, err := initObjectLayer(endpoints)
+	fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
 	if err != nil {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
@@ -159,11 +146,7 @@ func TestInitEventNotifierWithWebHook(t *testing.T) {
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
-	endpoints, err := parseStorageEndpoints(disks)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fs, _, err := initObjectLayer(endpoints)
+	fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
 	if err != nil {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
@@ -190,11 +173,7 @@ func TestInitEventNotifierWithAMQP(t *testing.T) {
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
-	endpoints, err := parseStorageEndpoints(disks)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fs, _, err := initObjectLayer(endpoints)
+	fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
 	if err != nil {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
@@ -221,11 +200,7 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) {
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
-	endpoints, err := parseStorageEndpoints(disks)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fs, _, err := initObjectLayer(endpoints)
+	fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
 	if err != nil {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
@@ -252,11 +227,7 @@ func TestInitEventNotifierWithRedis(t *testing.T) {
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
-	endpoints, err := parseStorageEndpoints(disks)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fs, _, err := initObjectLayer(endpoints)
+	fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
 	if err != nil {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
@@ -276,15 +247,10 @@ func (s *TestPeerRPCServerData) Setup(t *testing.T) {
 	s.testServer = StartTestPeersRPCServer(t, s.serverType)
 
 	// setup port and minio addr
-	host, port, err := net.SplitHostPort(s.testServer.Server.Listener.Addr().String())
-	if err != nil {
-		t.Fatalf("Initialisation error: %v", err)
-	}
+	host, port := mustSplitHostPort(s.testServer.Server.Listener.Addr().String())
 	globalMinioHost = host
 	globalMinioPort = port
-	globalMinioAddr = getLocalAddress(
-		s.testServer.SrvCmdCfg,
-	)
+	globalMinioAddr = getEndpointsLocalAddr(s.testServer.endpoints)
 
 	// initialize the peer client(s)
 	initGlobalS3Peers(s.testServer.Disks)
@@ -346,7 +312,7 @@ func TestInitEventNotifier(t *testing.T) {
 	filterRules := []filterRule{
 		{
 			Name:  "prefix",
-			Value: globalMinioDefaultOwnerID,
+			Value: "minio",
 		},
 		{
 			Name:  "suffix",
@@ -377,7 +343,7 @@ func TestInitEventNotifier(t *testing.T) {
 	}
 
 	// create bucket
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected error:", err)
 	}
 
@@ -442,7 +408,7 @@ func TestListenBucketNotification(t *testing.T) {
 	objectName := "object"
 
 	// Create the bucket to listen on
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected error:", err)
 	}
 
@@ -552,12 +518,12 @@ func TestAddRemoveBucketListenerConfig(t *testing.T) {
 
 	// Make a bucket to store topicConfigs.
 	randBucket := getRandomBucketName()
-	if err := obj.MakeBucket(randBucket); err != nil {
+	if err := obj.MakeBucketWithLocation(randBucket, ""); err != nil {
 		t.Fatalf("Failed to make bucket %s", randBucket)
 	}
 
 	// Add a topicConfig to an empty notificationConfig.
-	accountID := fmt.Sprintf("%d", time.Now().UTC().UnixNano())
+	accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
 	accountARN := fmt.Sprintf(
 		"arn:minio:sqs:%s:%s:listen-%s",
 		serverConfig.GetRegion(),
@@ -569,7 +535,7 @@ func TestAddRemoveBucketListenerConfig(t *testing.T) {
 	filterRules := []filterRule{
 		{
 			Name:  "prefix",
-			Value: globalMinioDefaultOwnerID,
+			Value: "minio",
 		},
 		{
 			Name:  "suffix",

cmd/file-logger.go

@@ -0,0 +1,86 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// FileLogger - file logger which logs to a file.
+type FileLogger struct {
+	BaseLogTarget
+	Filename string `json:"filename"`
+	file     *os.File
+}
+
+// Fire - log entry handler.
+func (logger FileLogger) Fire(entry *logrus.Entry) (err error) {
+	if !logger.Enable {
+		return nil
+	}
+
+	msgBytes, err := logger.formatter.Format(entry)
+	if err != nil {
+		return err
+	}
+
+	if _, err = logger.file.Write(msgBytes); err != nil {
+		return err
+	}
+
+	err = logger.file.Sync()
+	return err
+}
+
+// String - represents ConsoleLogger as string.
+func (logger FileLogger) String() string {
+	enableStr := "disabled"
+	if logger.Enable {
+		enableStr = "enabled"
+	}
+
+	return fmt.Sprintf("file:%s:%s", enableStr, logger.Filename)
+}
+
+// NewFileLogger - creates new file logger object.
+func NewFileLogger(filename string) (logger FileLogger) {
+	logger.Enable = true
+	logger.formatter = new(logrus.JSONFormatter)
+	logger.Filename = filename
+
+	return logger
+}
+
+// InitFileLogger - initializes file logger.
+func InitFileLogger(logger *FileLogger) (err error) {
+	if !logger.Enable {
+		return err
+	}
+
+	if logger.formatter == nil {
+		logger.formatter = new(logrus.JSONFormatter)
+	}
+
+	if logger.file == nil {
+		logger.file, err = os.OpenFile(logger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0664)
+	}
+
+	return err
+}

cmd/format-config-v1.go

@@ -20,8 +20,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"reflect"
 	"sync"
+
+	"github.com/minio/minio/pkg/lock"
 )
 
 // fsFormat - structure holding 'fs' format.
@@ -29,6 +33,15 @@ type fsFormat struct {
 	Version string `json:"version"`
 }
 
+// FS format version strings.
+const (
+	// Represents the current backend disk structure
+	// version under `.minio.sys` and actual data namespace.
+
+	// formatConfigV1.fsFormat.Version
+	fsFormatBackendV1 = "1"
+)
+
 // xlFormat - structure holding 'xl' format.
 type xlFormat struct {
 	Version string `json:"version"` // Version of 'xl' format.
@@ -38,6 +51,15 @@ type xlFormat struct {
 	JBOD []string `json:"jbod"`
 }
 
+// XL format version strings.
+const (
+	// Represents the current backend disk structure
+	// version under `.minio.sys` and actual data namespace.
+
+	// formatConfigV1.xlFormat.Version
+	xlFormatBackendV1 = "1"
+)
+
 // formatConfigV1 - structure holds format config version '1'.
 type formatConfigV1 struct {
 	Version string `json:"version"` // Version of the format config.
@@ -47,6 +69,120 @@ type formatConfigV1 struct {
 	XL     *xlFormat `json:"xl,omitempty"` // XL field holds xl format.
 }
 
+// Format json file.
+const (
+	// Format config file carries backend format specific details.
+	formatConfigFile = "format.json"
+
+	// Format config tmp file carries backend format.
+	formatConfigFileTmp = "format.json.tmp"
+)
+
+// `format.json` version value.
+const (
+	// formatConfigV1.Version represents the version string
+	// of the current structure and its fields in `format.json`.
+	formatFileV1 = "1"
+
+	// Future `format.json` structure changes should have
+	// its own version and should be subsequently listed here.
+)
+
+// Constitutes `format.json` backend name.
+const (
+	// Represents FS backend.
+	formatBackendFS = "fs"
+
+	// Represents XL backend.
+	formatBackendXL = "xl"
+)
+
+// CheckFS if the format is FS and is valid with right values
+// returns appropriate errors otherwise.
+func (f *formatConfigV1) CheckFS() error {
+	// Validate if format config version is v1.
+	if f.Version != formatFileV1 {
+		return fmt.Errorf("Unknown format file version '%s'", f.Version)
+	}
+
+	// Validate if we have the expected format.
+	if f.Format != formatBackendFS {
+		return fmt.Errorf("FS backend format required. Found '%s'", f.Format)
+	}
+
+	// Check if format is currently supported.
+	if f.FS.Version != fsFormatBackendV1 {
+		return fmt.Errorf("Unknown backend FS format version '%s'", f.FS.Version)
+	}
+
+	// Success.
+	return nil
+}
+
+// LoadFormat - loads format config v1, returns `errUnformattedDisk`
+// if reading format.json fails with io.EOF.
+func (f *formatConfigV1) LoadFormat(lk *lock.LockedFile) error {
+	_, err := f.ReadFrom(lk)
+	if errorCause(err) == io.EOF {
+		// No data on disk `format.json` still empty
+		// treat it as unformatted disk.
+		return traceError(errUnformattedDisk)
+	}
+	return err
+}
+
+func (f *formatConfigV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
+	// Serialize to prepare to write to disk.
+	var fbytes []byte
+	fbytes, err = json.Marshal(f)
+	if err != nil {
+		return 0, traceError(err)
+	}
+	if err = lk.Truncate(0); err != nil {
+		return 0, traceError(err)
+	}
+	_, err = lk.Write(fbytes)
+	if err != nil {
+		return 0, traceError(err)
+	}
+	return int64(len(fbytes)), nil
+}
+
+func (f *formatConfigV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
+	var fbytes []byte
+	fi, err := lk.Stat()
+	if err != nil {
+		return 0, traceError(err)
+	}
+	fbytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
+	if err != nil {
+		return 0, traceError(err)
+	}
+	if len(fbytes) == 0 {
+		return 0, traceError(io.EOF)
+	}
+	// Decode `format.json`.
+	if err = json.Unmarshal(fbytes, f); err != nil {
+		return 0, traceError(err)
+	}
+	return int64(len(fbytes)), nil
+}
+
+func newFSFormat() (format *formatConfigV1) {
+	return newFSFormatV1()
+}
+
+// newFSFormatV1 - initializes new formatConfigV1 with FS format info.
+func newFSFormatV1() (format *formatConfigV1) {
+	return &formatConfigV1{
+		Version: formatFileV1,
+		Format:  formatBackendFS,
+		FS: &fsFormat{
+			Version: fsFormatBackendV1,
+		},
+	}
+}
+
 /*
 
 All disks online
@@ -768,31 +904,39 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
 	return reorderDisks(bootstrapDisks, formatConfigs)
 }
 
-func checkFormatXLValues(formatConfigs []*formatConfigV1) error {
-	for _, formatXL := range formatConfigs {
-		if formatXL == nil {
-			continue
-		}
-		// Validate format version and format type.
-		if formatXL.Version != "1" {
-			return fmt.Errorf("Unsupported version of backend format [%s] found", formatXL.Version)
-		}
-		if formatXL.Format != "xl" {
-			return fmt.Errorf("Unsupported backend format [%s] found", formatXL.Format)
-		}
-		if formatXL.XL.Version != "1" {
-			return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version)
-		}
-		if len(formatConfigs) != len(formatXL.XL.JBOD) {
-			return fmt.Errorf("Number of disks %d did not match the backend format %d", len(formatConfigs), len(formatXL.XL.JBOD))
-		}
+func checkFormatXLValue(formatXL *formatConfigV1) error {
+	// Validate format version and format type.
+	if formatXL.Version != formatFileV1 {
+		return fmt.Errorf("Unsupported version of backend format [%s] found", formatXL.Version)
+	}
+	if formatXL.Format != formatBackendXL {
+		return fmt.Errorf("Unsupported backend format [%s] found", formatXL.Format)
+	}
+	if formatXL.XL.Version != "1" {
+		return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version)
 	}
 	return nil
 }
 
+func checkFormatXLValues(formatConfigs []*formatConfigV1) (int, error) {
+	for i, formatXL := range formatConfigs {
+		if formatXL == nil {
+			continue
+		}
+		if err := checkFormatXLValue(formatXL); err != nil {
+			return i, err
+		}
+		if len(formatConfigs) != len(formatXL.XL.JBOD) {
+			return i, fmt.Errorf("Number of disks %d did not match the backend format %d",
+				len(formatConfigs), len(formatXL.XL.JBOD))
+		}
+	}
+	return -1, nil
+}
+
 // checkFormatXL - verifies if format.json format is intact.
 func checkFormatXL(formatConfigs []*formatConfigV1) error {
-	if err := checkFormatXLValues(formatConfigs); err != nil {
+	if _, err := checkFormatXLValues(formatConfigs); err != nil {
 		return err
 	}
 	if err := checkJBODConsistency(formatConfigs); err != nil {
@@ -867,10 +1011,10 @@ func initFormatXL(storageDisks []StorageAPI) (err error) {
 		}
 		// Allocate format config.
 		formats[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    mustGetUUID(),
 			},
 		}

cmd/format-config-v1_test.go

@@ -18,7 +18,12 @@ package cmd
 
 import (
 	"bytes"
+	"errors"
+	"os"
+	"path/filepath"
 	"testing"
+
+	"github.com/minio/minio/pkg/lock"
 )
 
 // generates a valid format.json for XL backend.
@@ -30,10 +35,10 @@ func genFormatXLValid() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -51,10 +56,10 @@ func genFormatXLInvalidVersion() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -75,10 +80,10 @@ func genFormatXLInvalidFormat() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -99,10 +104,10 @@ func genFormatXLInvalidXLVersion() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -116,8 +121,8 @@ func genFormatXLInvalidXLVersion() []*formatConfigV1 {
 
 func genFormatFS() *formatConfigV1 {
 	return &formatConfigV1{
-		Version: "1",
-		Format:  "fs",
+		Version: formatFileV1,
+		Format:  formatBackendFS,
 	}
 }
 
@@ -130,10 +135,10 @@ func genFormatXLInvalidJBODCount() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -151,10 +156,10 @@ func genFormatXLInvalidJBOD() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -178,10 +183,10 @@ func genFormatXLInvalidDisks() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -202,10 +207,10 @@ func genFormatXLInvalidDisksOrder() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -223,7 +228,7 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
 	var err error
 	xl := obj.(*xlObjects)
 
-	err = obj.MakeBucket("bucket")
+	err = obj.MakeBucketWithLocation("bucket", "")
 	if err != nil {
 		return []StorageAPI{}, err
 	}
@@ -240,10 +245,10 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
 	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
 	// created when minio starts
 	for i := 3; i <= 5; i++ {
-		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 			return []StorageAPI{}, err
 		}
-		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "tmp"); err != nil {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, "tmp"); err != nil {
 			return []StorageAPI{}, err
 		}
 		if err = xl.storageDisks[i].DeleteFile(bucket, object+"/xl.json"); err != nil {
@@ -273,12 +278,8 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	endpoints, err := parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
 	// Create an instance of xl backend.
-	obj, _, err := initObjectLayer(endpoints)
+	obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Error(err)
 	}
@@ -309,12 +310,8 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	endpoints, err := parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
 	// Create an instance of xl backend.
-	obj, _, err := initObjectLayer(endpoints)
+	obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Error(err)
 	}
@@ -354,7 +351,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 
 	xl := obj.(*xlObjects)
 
-	err = obj.MakeBucket("bucket")
+	err = obj.MakeBucketWithLocation("bucket", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -369,19 +366,19 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 	}
 
 	// Now, remove two format files.. Load them and reorder
-	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
+	if err = xl.storageDisks[3].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 		t.Fatal(err)
 	}
-	if err = xl.storageDisks[11].DeleteFile(".minio.sys", "format.json"); err != nil {
+	if err = xl.storageDisks[11].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 		t.Fatal(err)
 	}
 
 	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
 	// created when minio starts
-	if err = xl.storageDisks[10].DeleteFile(".minio.sys", "format.json"); err != nil {
+	if err = xl.storageDisks[10].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 		t.Fatal(err)
 	}
-	if err = xl.storageDisks[10].DeleteFile(".minio.sys", "tmp"); err != nil {
+	if err = xl.storageDisks[10].DeleteFile(minioMetaBucket, "tmp"); err != nil {
 		t.Fatal(err)
 	}
 	if err = xl.storageDisks[10].DeleteFile(bucket, object+"/xl.json"); err != nil {
@@ -427,7 +424,7 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 
 	xl := obj.(*xlObjects)
 
-	err = obj.MakeBucket("bucket")
+	err = obj.MakeBucketWithLocation("bucket", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -442,10 +439,10 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 	}
 
 	// Now, remove two format files.. Load them and reorder
-	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
+	if err = xl.storageDisks[3].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 		t.Fatal(err)
 	}
-	if err = xl.storageDisks[5].DeleteFile(".minio.sys", "format.json"); err != nil {
+	if err = xl.storageDisks[5].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 		t.Fatal(err)
 	}
 
@@ -563,10 +560,10 @@ func TestSavedUUIDOrder(t *testing.T) {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -596,12 +593,8 @@ func TestInitFormatXLErrors(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer removeRoots(fsDirs)
-	endpoints, err := parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
 	// Create an instance of xl backend.
-	obj, _, err := initObjectLayer(endpoints)
+	obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -694,6 +687,163 @@ func TestGenericFormatCheckXL(t *testing.T) {
 	}
 }
 
+// TestFSCheckFormatFSErr - test loadFormatFS loading older format.
+func TestFSCheckFormatFSErr(t *testing.T) {
+	// Prepare for testing
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer removeAll(disk)
+
+	// Assign a new UUID.
+	uuid := mustGetUUID()
+
+	// Initialize meta volume, if volume already exists ignores it.
+	if err := initMetaVolumeFS(disk, uuid); err != nil {
+		t.Fatal(err)
+	}
+
+	testCases := []struct {
+		format         *formatConfigV1
+		formatWriteErr error
+		formatCheckErr error
+		shouldPass     bool
+	}{
+		{
+			format: &formatConfigV1{
+				Version: formatFileV1,
+				Format:  formatBackendFS,
+				FS: &fsFormat{
+					Version: fsFormatBackendV1,
+				},
+			},
+			formatCheckErr: nil,
+			shouldPass:     true,
+		},
+		{
+			format: &formatConfigV1{
+				Version: formatFileV1,
+				Format:  formatBackendFS,
+				FS: &fsFormat{
+					Version: "10",
+				},
+			},
+			formatCheckErr: errors.New("Unknown backend FS format version '10'"),
+			shouldPass:     false,
+		},
+		{
+			format: &formatConfigV1{
+				Version: formatFileV1,
+				Format:  "garbage",
+				FS: &fsFormat{
+					Version: fsFormatBackendV1,
+				},
+			},
+			formatCheckErr: errors.New("FS backend format required. Found 'garbage'"),
+		},
+		{
+			format: &formatConfigV1{
+				Version: "-1",
+				Format:  formatBackendFS,
+				FS: &fsFormat{
+					Version: fsFormatBackendV1,
+				},
+			},
+			formatCheckErr: errors.New("Unknown format file version '-1'"),
+		},
+	}
+
+	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
+	for i, testCase := range testCases {
+		lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = testCase.format.WriteTo(lk)
+		lk.Close()
+		if err != nil {
+			t.Fatalf("Test %d: Expected nil, got %s", i+1, err)
+		}
+
+		lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		formatCfg := &formatConfigV1{}
+		_, err = formatCfg.ReadFrom(lk)
+		lk.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = formatCfg.CheckFS()
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Should not fail with unexpected %s, expected nil", i+1, err)
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Should fail with expected %s, got nil", i+1, testCase.formatCheckErr)
+		}
+		if err != nil && !testCase.shouldPass {
+			if errorCause(err).Error() != testCase.formatCheckErr.Error() {
+				t.Errorf("Test %d: Should fail with expected %s, got %s", i+1, testCase.formatCheckErr, err)
+			}
+		}
+	}
+}
+
+// TestFSCheckFormatFS - test loadFormatFS with healty and faulty disks
+func TestFSCheckFormatFS(t *testing.T) {
+	// Prepare for testing
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer removeAll(disk)
+
+	// Assign a new UUID.
+	uuid := mustGetUUID()
+
+	// Initialize meta volume, if volume already exists ignores it.
+	if err := initMetaVolumeFS(disk, uuid); err != nil {
+		t.Fatal(err)
+	}
+
+	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
+	lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	format := newFSFormatV1()
+	_, err = format.WriteTo(lk)
+	lk.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Loading corrupted format file
+	file, err := os.OpenFile(preparePath(fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		t.Fatal("Should not fail here", err)
+	}
+	file.Write([]byte{'b'})
+	file.Close()
+
+	lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	format = &formatConfigV1{}
+	_, err = format.ReadFrom(lk)
+	lk.Close()
+	if err == nil {
+		t.Fatal("Should return an error here")
+	}
+
+	// Loading format file from disk not found.
+	removeAll(disk)
+	_, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDONLY, 0600)
+	if err != nil && !os.IsNotExist(err) {
+		t.Fatal("Should return 'format.json' does not exist, but got", err)
+	}
+}
+
 func TestLoadFormatXLErrs(t *testing.T) {
 	nDisks := 16
 	fsDirs, err := getRandomDisks(nDisks)
@@ -702,12 +852,8 @@ func TestLoadFormatXLErrs(t *testing.T) {
 	}
 	defer removeRoots(fsDirs)
 
-	endpoints, err := parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
 	// Create an instance of xl backend.
-	obj, _, err := initObjectLayer(endpoints)
+	obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -733,11 +879,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
 	}
 	defer removeRoots(fsDirs)
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -761,11 +903,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
 	}
 	defer removeRoots(fsDirs)
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -773,7 +911,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
 
 	// disks 0..10 returns unformatted disk
 	for i := 0; i <= 10; i++ {
-		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -787,11 +925,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
 	}
 	defer removeRoots(fsDirs)
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -820,13 +954,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err := parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Everything is fine, should return nil
-	obj, _, err := initObjectLayer(endpoints)
+	obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -842,13 +971,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Disks 0..15 are nil
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -866,13 +990,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// One disk returns Faulty Disk
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -892,13 +1011,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// One disk is not found, heal corrupted disks should return nil
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -914,19 +1028,14 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Remove format.json of all disks
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
 	xl = obj.(*xlObjects)
 	for i := 0; i <= 15; i++ {
-		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -940,19 +1049,14 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Corrupted format json in one disk
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
 	xl = obj.(*xlObjects)
 	for i := 0; i <= 15; i++ {
-		if err = xl.storageDisks[i].AppendFile(".minio.sys", "format.json", []byte("corrupted data")); err != nil {
+		if err = xl.storageDisks[i].AppendFile(minioMetaBucket, formatConfigFile, []byte("corrupted data")); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -976,13 +1080,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err := parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Everything is fine, should return nil
-	obj, _, err := initObjectLayer(endpoints)
+	obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -997,13 +1096,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Disks 0..15 are nil
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1021,13 +1115,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// One disk returns Faulty Disk
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1047,13 +1136,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// One disk is not found, heal corrupted disks should return nil
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1069,19 +1153,14 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	endpoints, err = parseStorageEndpoints(fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// Remove format.json of all disks
-	obj, _, err = initObjectLayer(endpoints)
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
 	xl = obj.(*xlObjects)
 	for i := 0; i <= 15; i++ {
-		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 			t.Fatal(err)
 		}
 	}

cmd/fs-v1-helpers.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"os"
 	pathutil "path"
+	"runtime"
 )
 
 // Removes only the file at given path does not remove
@@ -122,18 +123,27 @@ func fsMkdir(dirPath string) (err error) {
 	return nil
 }
 
-// Lookup if directory exists, returns directory
-// attributes upon success.
-func fsStatDir(statDir string) (os.FileInfo, error) {
-	if statDir == "" {
+func fsStat(statLoc string) (os.FileInfo, error) {
+	if statLoc == "" {
 		return nil, traceError(errInvalidArgument)
 	}
-	if err := checkPathLength(statDir); err != nil {
+	if err := checkPathLength(statLoc); err != nil {
+		return nil, traceError(err)
+	}
+	fi, err := osStat(preparePath(statLoc))
+	if err != nil {
 		return nil, traceError(err)
 	}
 
-	fi, err := os.Stat(preparePath(statDir))
+	return fi, nil
+}
+
+// Lookup if directory exists, returns directory
+// attributes upon success.
+func fsStatDir(statDir string) (os.FileInfo, error) {
+	fi, err := fsStat(statDir)
 	if err != nil {
+		err = errorCause(err)
 		if os.IsNotExist(err) {
 			return nil, traceError(errVolumeNotFound)
 		} else if os.IsPermission(err) {
@@ -151,16 +161,9 @@ func fsStatDir(statDir string) (os.FileInfo, error) {
 
 // Lookup if file exists, returns file attributes upon success
 func fsStatFile(statFile string) (os.FileInfo, error) {
-	if statFile == "" {
-		return nil, traceError(errInvalidArgument)
-	}
-
-	if err := checkPathLength(statFile); err != nil {
-		return nil, traceError(err)
-	}
-
-	fi, err := os.Stat(preparePath(statFile))
+	fi, err := fsStat(statFile)
 	if err != nil {
+		err = errorCause(err)
 		if os.IsNotExist(err) {
 			return nil, traceError(errFileNotFound)
 		} else if os.IsPermission(err) {
@@ -173,7 +176,7 @@ func fsStatFile(statFile string) (os.FileInfo, error) {
 		return nil, traceError(err)
 	}
 	if fi.IsDir() {
-		return nil, traceError(errFileNotFound)
+		return nil, traceError(errFileAccessDenied)
 	}
 	return fi, nil
 }
@@ -205,7 +208,7 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
 	}
 
 	// Stat to get the size of the file at path.
-	st, err := fr.Stat()
+	st, err := osStat(preparePath(readPath))
 	if err != nil {
 		return nil, 0, traceError(err)
 	}
@@ -229,7 +232,7 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
 
 // Creates a file and copies data from incoming reader. Staging buffer is used by io.CopyBuffer.
 func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int64) (int64, error) {
-	if filePath == "" || reader == nil || buf == nil {
+	if filePath == "" || reader == nil {
 		return 0, traceError(errInvalidArgument)
 	}
 
@@ -262,11 +265,18 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
 		}
 	}
 
-	bytesWritten, err := io.CopyBuffer(writer, reader, buf)
-	if err != nil {
-		return 0, traceError(err)
+	var bytesWritten int64
+	if buf != nil {
+		bytesWritten, err = io.CopyBuffer(writer, reader, buf)
+		if err != nil {
+			return 0, traceError(err)
+		}
+	} else {
+		bytesWritten, err = io.Copy(writer, reader)
+		if err != nil {
+			return 0, traceError(err)
+		}
 	}
-
 	return bytesWritten, nil
 }
 
@@ -275,6 +285,12 @@ func fsRemoveUploadIDPath(basePath, uploadIDPath string) error {
 	if basePath == "" || uploadIDPath == "" {
 		return traceError(errInvalidArgument)
 	}
+	if err := checkPathLength(basePath); err != nil {
+		return traceError(err)
+	}
+	if err := checkPathLength(uploadIDPath); err != nil {
+		return traceError(err)
+	}
 
 	// List all the entries in uploadID.
 	entries, err := readDir(uploadIDPath)
@@ -318,6 +334,26 @@ func fsFAllocate(fd int, offset int64, len int64) (err error) {
 // Renames source path to destination path, creates all the
 // missing parents if they don't exist.
 func fsRenameFile(sourcePath, destPath string) error {
+	if err := checkPathLength(sourcePath); err != nil {
+		return traceError(err)
+	}
+	if err := checkPathLength(destPath); err != nil {
+		return traceError(err)
+	}
+	// Verify if source path exists.
+	if _, err := os.Stat(preparePath(sourcePath)); err != nil {
+		if os.IsNotExist(err) {
+			return traceError(errFileNotFound)
+		} else if os.IsPermission(err) {
+			return traceError(errFileAccessDenied)
+		} else if isSysErrPathNotFound(err) {
+			return traceError(errFileNotFound)
+		} else if isSysErrNotDir(err) {
+			// File path cannot be verified since one of the parents is a file.
+			return traceError(errFileAccessDenied)
+		}
+		return traceError(err)
+	}
 	if err := mkdirAll(pathutil.Dir(destPath), 0777); err != nil {
 		return traceError(err)
 	}
@@ -327,8 +363,7 @@ func fsRenameFile(sourcePath, destPath string) error {
 	return nil
 }
 
-// Delete a file and its parent if it is empty at the destination path.
-// this function additionally protects the basePath from being deleted.
+// fsDeleteFile is a wrapper for deleteFile(), after checking the path length.
 func fsDeleteFile(basePath, deletePath string) error {
 	if err := checkPathLength(basePath); err != nil {
 		return traceError(err)
@@ -338,42 +373,51 @@ func fsDeleteFile(basePath, deletePath string) error {
 		return traceError(err)
 	}
 
-	if basePath == deletePath {
-		return nil
-	}
-
-	// Verify if the path exists.
-	pathSt, err := os.Stat(preparePath(deletePath))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return traceError(errFileNotFound)
-		} else if os.IsPermission(err) {
-			return traceError(errFileAccessDenied)
-		}
-		return traceError(err)
-	}
-
-	if pathSt.IsDir() && !isDirEmpty(deletePath) {
-		// Verify if directory is empty.
-		return nil
-	}
-
-	// Attempt to remove path.
-	if err = os.Remove(preparePath(deletePath)); err != nil {
-		if os.IsNotExist(err) {
-			return traceError(errFileNotFound)
-		} else if os.IsPermission(err) {
-			return traceError(errFileAccessDenied)
-		} else if isSysErrNotEmpty(err) {
-			return traceError(errVolumeNotEmpty)
-		}
-		return traceError(err)
-	}
-
-	// Recursively go down the next path and delete again.
-	if err := fsDeleteFile(basePath, pathutil.Dir(deletePath)); err != nil {
-		return err
-	}
-
-	return nil
+	return deleteFile(basePath, deletePath)
+}
+
+// fsRemoveMeta safely removes a locked file and takes care of Windows special case
+func fsRemoveMeta(basePath, deletePath, tmpDir string) error {
+	// Special case for windows please read through.
+	if runtime.GOOS == globalWindowsOSName {
+		// Ordinarily windows does not permit deletion or renaming of files still
+		// in use, but if all open handles to that file were opened with FILE_SHARE_DELETE
+		// then it can permit renames and deletions of open files.
+		//
+		// There are however some gotchas with this, and it is worth listing them here.
+		// Firstly, Windows never allows you to really delete an open file, rather it is
+		// flagged as delete pending and its entry in its directory remains visible
+		// (though no new file handles may be opened to it) and when the very last
+		// open handle to the file in the system is closed, only then is it truly
+		// deleted. Well, actually only sort of truly deleted, because Windows only
+		// appears to remove the file entry from the directory, but in fact that
+		// entry is merely hidden and actually still exists and attempting to create
+		// a file with the same name will return an access denied error. How long it
+		// silently exists for depends on a range of factors, but put it this way:
+		// if your code loops creating and deleting the same file name as you might
+		// when operating a lock file, you're going to see lots of random spurious
+		// access denied errors and truly dismal lock file performance compared to POSIX.
+		//
+		// We work-around these un-POSIX file semantics by taking a dual step to
+		// deleting files. Firstly, it renames the file to tmp location into multipartTmpBucket
+		// We always open files with FILE_SHARE_DELETE permission enabled, with that
+		// flag Windows permits renaming and deletion, and because the name was changed
+		// to a very random name somewhere not in its origin directory before deletion,
+		// you don't see those unexpected random errors when creating files with the
+		// same name as a recently deleted file as you do anywhere else on Windows.
+		// Because the file is probably not in its original containing directory any more,
+		// deletions of that directory will not fail with "directory not empty" as they
+		// otherwise normally would either.
+
+		tmpPath := pathJoin(tmpDir, mustGetUUID())
+
+		fsRenameFile(deletePath, tmpPath)
+
+		// Proceed to deleting the directory if empty
+		fsDeleteFile(basePath, pathutil.Dir(deletePath))
+
+		// Finally delete the renamed file.
+		return fsDeleteFile(tmpDir, tmpPath)
+	}
+	return fsDeleteFile(basePath, deletePath)
 }

cmd/fs-v1-helpers_test.go

@@ -18,9 +18,40 @@ package cmd
 
 import (
 	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
 	"testing"
+
+	"github.com/minio/minio/pkg/lock"
 )
 
+func TestFSRenameFile(t *testing.T) {
+	// create posix test setup
+	_, path, err := newPosixTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer removeAll(path)
+
+	if err = fsMkdir(pathJoin(path, "testvolume1")); err != nil {
+		t.Fatal(err)
+	}
+	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil {
+		t.Fatal(err)
+	}
+	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNotFound {
+		t.Fatal(err)
+	}
+	if err = fsRenameFile(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNameTooLong {
+		t.Fatal("Unexpected error", err)
+	}
+	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
+		t.Fatal("Unexpected error", err)
+	}
+}
+
 func TestFSStats(t *testing.T) {
 	// create posix test setup
 	_, path, err := newPosixTestSetup()
@@ -43,9 +74,8 @@ func TestFSStats(t *testing.T) {
 		t.Fatalf("Unable to create volume, %s", err)
 	}
 
-	var buf = make([]byte, 4096)
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -55,7 +85,7 @@ func TestFSStats(t *testing.T) {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if _, err = fsCreateFile(pathJoin(path, "success-vol", "path/to/success-file"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(pathJoin(path, "success-vol", "path/to/success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -105,7 +135,7 @@ func TestFSStats(t *testing.T) {
 			srcFSPath:   path,
 			srcVol:      "success-vol",
 			srcPath:     "path",
-			expectedErr: errFileNotFound,
+			expectedErr: errFileAccessDenied,
 		},
 		// Test case - 6.
 		// Test case with src path segment > 255.
@@ -138,7 +168,8 @@ func TestFSStats(t *testing.T) {
 
 	for i, testCase := range testCases {
 		if testCase.srcPath != "" {
-			if _, err := fsStatFile(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
+			if _, err := fsStatFile(pathJoin(testCase.srcFSPath, testCase.srcVol,
+				testCase.srcPath)); errorCause(err) != testCase.expectedErr {
 				t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 			}
 		} else {
@@ -169,9 +200,8 @@ func TestFSCreateAndOpen(t *testing.T) {
 		t.Fatal("Unexpected error", err)
 	}
 
-	var buf = make([]byte, 4096)
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -199,7 +229,7 @@ func TestFSCreateAndOpen(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		_, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, buf, reader.Size())
+		_, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0)
 		if errorCause(err) != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
@@ -234,50 +264,123 @@ func TestFSDeletes(t *testing.T) {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
-	reader.Seek(0, 0)
+	reader.Seek(0, io.SeekStart)
+
+	// folder is not empty
+	err = fsMkdir(pathJoin(path, "success-vol", "not-empty"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ioutil.WriteFile(pathJoin(path, "success-vol", "not-empty", "file"), []byte("data"), 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// recursive
+	if err = fsMkdir(pathJoin(path, "success-vol", "parent")); err != nil {
+		t.Fatal(err)
+	}
+	if err = fsMkdir(pathJoin(path, "success-vol", "parent", "dir")); err != nil {
+		t.Fatal(err)
+	}
 
 	testCases := []struct {
+		basePath    string
 		srcVol      string
 		srcPath     string
 		expectedErr error
 	}{
-		// Test case - 1.
 		// valid case with existing volume and file to delete.
 		{
+			basePath:    path,
 			srcVol:      "success-vol",
 			srcPath:     "success-file",
 			expectedErr: nil,
 		},
-		// Test case - 2.
 		// The file was deleted in the last case, so DeleteFile should fail.
 		{
+			basePath:    path,
 			srcVol:      "success-vol",
 			srcPath:     "success-file",
 			expectedErr: errFileNotFound,
 		},
-		// Test case - 3.
 		// Test case with segment of the volume name > 255.
 		{
+			basePath:    path,
 			srcVol:      "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
 			srcPath:     "success-file",
 			expectedErr: errFileNameTooLong,
 		},
-		// Test case - 4.
 		// Test case with src path segment > 255.
 		{
+			basePath:    path,
 			srcVol:      "success-vol",
 			srcPath:     "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
 			expectedErr: errFileNameTooLong,
 		},
+		// Base path is way too long.
+		{
+			basePath:    "path03333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333",
+			srcVol:      "success-vol",
+			srcPath:     "object",
+			expectedErr: errFileNameTooLong,
+		},
+		// Directory is not empty. Should give nil, but won't delete.
+		{
+			basePath:    path,
+			srcVol:      "success-vol",
+			srcPath:     "not-empty",
+			expectedErr: nil,
+		},
+		// Should delete recursively.
+		{
+			basePath:    path,
+			srcVol:      "success-vol",
+			srcPath:     pathJoin("parent", "dir"),
+			expectedErr: nil,
+		},
 	}
 
 	for i, testCase := range testCases {
-		if err = fsDeleteFile(path, pathJoin(path, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
+		if err = fsDeleteFile(testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
 	}
 }
 
+func BenchmarkFSDeleteFile(b *testing.B) {
+	// create posix test setup
+	_, path, err := newPosixTestSetup()
+	if err != nil {
+		b.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer removeAll(path)
+
+	// Setup test environment.
+	if err = fsMkdir(pathJoin(path, "benchmark")); err != nil {
+		b.Fatalf("Unable to create directory, %s", err)
+	}
+
+	benchDir := pathJoin(path, "benchmark")
+	filename := pathJoin(benchDir, "file.txt")
+
+	b.ResetTimer()
+	// We need to create and delete the file sequentially inside the benchmark.
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		err = ioutil.WriteFile(filename, []byte("data"), 0777)
+		if err != nil {
+			b.Fatal(err)
+		}
+		b.StartTimer()
+
+		err = fsDeleteFile(benchDir, filename)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 // Tests fs removes.
 func TestFSRemoves(t *testing.T) {
 	// create posix test setup
@@ -292,15 +395,14 @@ func TestFSRemoves(t *testing.T) {
 		t.Fatalf("Unable to create directory, %s", err)
 	}
 
-	var buf = make([]byte, 4096)
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
 	reader.Seek(0, 0)
 
-	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file-new"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file-new"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -396,3 +498,51 @@ func TestFSRemoves(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestFSRemoveMeta(t *testing.T) {
+	// create posix test setup
+	_, fsPath, err := newPosixTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer removeAll(fsPath)
+
+	// Setup test environment.
+	if err = fsMkdir(pathJoin(fsPath, "success-vol")); err != nil {
+		t.Fatalf("Unable to create directory, %s", err)
+	}
+
+	filePath := pathJoin(fsPath, "success-vol", "success-file")
+
+	var reader = bytes.NewReader([]byte("Hello, world"))
+	if _, err = fsCreateFile(filePath, reader, nil, 0); err != nil {
+		t.Fatalf("Unable to create file, %s", err)
+	}
+
+	rwPool := &fsIOPool{
+		readersMap: make(map[string]*lock.RLockedFile),
+	}
+
+	if _, err := rwPool.Open(filePath); err != nil {
+		t.Fatalf("Unable to lock file %s", filePath)
+	}
+
+	defer rwPool.Close(filePath)
+
+	tmpDir, tmpErr := ioutil.TempDir(globalTestTmpDir, "minio-")
+	if tmpErr != nil {
+		t.Fatal(tmpErr)
+	}
+
+	if err := fsRemoveMeta(fsPath, filePath, tmpDir); err != nil {
+		t.Fatalf("Unable to remove file, %s", err)
+	}
+
+	if _, err := osStat(preparePath(filePath)); !os.IsNotExist(err) {
+		t.Fatalf("`%s` file found though it should have been deleted.", filePath)
+	}
+
+	if _, err := osStat(preparePath(path.Dir(filePath))); !os.IsNotExist(err) {
+		t.Fatalf("`%s` parent directory found though it should have been deleted.", filePath)
+	}
+}

cmd/fs-v1-metadata.go

@@ -24,14 +24,31 @@ import (
 	pathutil "path"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/mimedb"
+	"github.com/tidwall/gjson"
 )
 
+// FS format, and object metadata.
 const (
-	fsMetaJSONFile   = "fs.json"
-	fsFormatJSONFile = "format.json"
+	// fs.json object metadata.
+	fsMetaJSONFile = "fs.json"
+)
+
+// FS metadata constants.
+const (
+	// FS backend meta 1.0.0 version.
+	fsMetaVersion100 = "1.0.0"
+
+	// FS backend meta 1.0.1 version.
+	fsMetaVersion = "1.0.1"
+
+	// FS backend meta format.
+	fsMetaFormat = "fs"
+
+	// Add more constants here.
 )
 
 // A fsMetaV1 represents a metadata header mapping keys to sets of values.
@@ -46,6 +63,19 @@ type fsMetaV1 struct {
 	Parts []objectPartInfo  `json:"parts,omitempty"`
 }
 
+// IsValid - tells if the format is sane by validating the version
+// string and format style.
+func (m fsMetaV1) IsValid() bool {
+	return isFSMetaValid(m.Version, m.Format)
+}
+
+// Verifies if the backend format metadata is sane by validating
+// the version string and format style.
+func isFSMetaValid(version, format string) bool {
+	return ((version == fsMetaVersion || version == fsMetaVersion100) &&
+		format == fsMetaFormat)
+}
+
 // Converts metadata to object info.
 func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo {
 	if len(m.Meta) == 0 {
@@ -66,7 +96,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 		Name:   object,
 	}
 
-	// We set file into only if its valid.
+	// We set file info only if its valid.
 	objInfo.ModTime = timeSentinel
 	if fi != nil {
 		objInfo.ModTime = fi.ModTime()
@@ -74,17 +104,15 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 		objInfo.IsDir = fi.IsDir()
 	}
 
-	objInfo.MD5Sum = m.Meta["md5Sum"]
+	// Extract etag from metadata.
+	objInfo.ETag = extractETag(m.Meta)
 	objInfo.ContentType = m.Meta["content-type"]
 	objInfo.ContentEncoding = m.Meta["content-encoding"]
 
-	// md5Sum has already been extracted into objInfo.MD5Sum.  We
-	// need to remove it from m.Meta to avoid it from appearing as
-	// part of response headers. e.g, X-Minio-* or X-Amz-*.
-	delete(m.Meta, "md5Sum")
-
-	// Save all the other userdefined API.
-	objInfo.UserDefined = m.Meta
+	// etag/md5Sum has already been extracted. We need to
+	// remove to avoid it from appearing as part of
+	// response headers. e.g, X-Minio-* or X-Amz-*.
+	objInfo.UserDefined = cleanMetaETag(m.Meta)
 
 	// Success..
 	return objInfo
@@ -144,42 +172,84 @@ func (m *fsMetaV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
 	return int64(len(metadataBytes)), nil
 }
 
+func parseFSVersion(fsMetaBuf []byte) string {
+	return gjson.GetBytes(fsMetaBuf, "version").String()
+}
+
+func parseFSFormat(fsMetaBuf []byte) string {
+	return gjson.GetBytes(fsMetaBuf, "format").String()
+}
+
+func parseFSRelease(fsMetaBuf []byte) string {
+	return gjson.GetBytes(fsMetaBuf, "minio.release").String()
+}
+
+func parseFSMetaMap(fsMetaBuf []byte) map[string]string {
+	// Get xlMetaV1.Meta map.
+	metaMapResult := gjson.GetBytes(fsMetaBuf, "meta").Map()
+	metaMap := make(map[string]string)
+	for key, valResult := range metaMapResult {
+		metaMap[key] = valResult.String()
+	}
+	return metaMap
+}
+
+func parseFSParts(fsMetaBuf []byte) []objectPartInfo {
+	// Parse the FS Parts.
+	partsResult := gjson.GetBytes(fsMetaBuf, "parts").Array()
+	partInfo := make([]objectPartInfo, len(partsResult))
+	for i, p := range partsResult {
+		info := objectPartInfo{}
+		info.Number = int(p.Get("number").Int())
+		info.Name = p.Get("name").String()
+		info.ETag = p.Get("etag").String()
+		info.Size = p.Get("size").Int()
+		partInfo[i] = info
+	}
+	return partInfo
+}
+
 func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
-	var metadataBytes []byte
+	var fsMetaBuf []byte
 	fi, err := lk.Stat()
 	if err != nil {
 		return 0, traceError(err)
 	}
 
-	metadataBytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
+	fsMetaBuf, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
 	if err != nil {
 		return 0, traceError(err)
 	}
 
-	if len(metadataBytes) == 0 {
+	if len(fsMetaBuf) == 0 {
 		return 0, traceError(io.EOF)
 	}
 
-	// Decode `fs.json` into fsMeta structure.
-	if err = json.Unmarshal(metadataBytes, m); err != nil {
-		return 0, traceError(err)
+	// obtain version.
+	m.Version = parseFSVersion(fsMetaBuf)
+
+	// obtain format.
+	m.Format = parseFSFormat(fsMetaBuf)
+
+	// Verify if the format is valid, return corrupted format
+	// for unrecognized formats.
+	if !isFSMetaValid(m.Version, m.Format) {
+		return 0, traceError(errCorruptedFormat)
 	}
 
+	// obtain metadata.
+	m.Meta = parseFSMetaMap(fsMetaBuf)
+
+	// obtain parts info list.
+	m.Parts = parseFSParts(fsMetaBuf)
+
+	// obtain minio release date.
+	m.Minio.Release = parseFSRelease(fsMetaBuf)
+
 	// Success.
-	return int64(len(metadataBytes)), nil
+	return int64(len(fsMetaBuf)), nil
 }
 
-// FS metadata constants.
-const (
-	// FS backend meta version.
-	fsMetaVersion = "1.0.0"
-
-	// FS backend meta format.
-	fsMetaFormat = "fs"
-
-	// Add more constants here.
-)
-
 // newFSMetaV1 - initializes new fsMetaV1.
 func newFSMetaV1() (fsMeta fsMetaV1) {
 	fsMeta = fsMetaV1{}
@@ -189,60 +259,97 @@ func newFSMetaV1() (fsMeta fsMetaV1) {
 	return fsMeta
 }
 
-// newFSFormatV1 - initializes new formatConfigV1 with FS format info.
-func newFSFormatV1() (format *formatConfigV1) {
-	return &formatConfigV1{
-		Version: "1",
-		Format:  "fs",
-		FS: &fsFormat{
-			Version: "1",
-		},
-	}
-}
+// Check if disk has already a valid format, holds a read lock and
+// upon success returns it to the caller to be closed.
+func checkLockedValidFormatFS(fsPath string) (*lock.RLockedFile, error) {
+	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
 
-// loads format.json from minioMetaBucket if it exists.
-func loadFormatFS(fsPath string) (*formatConfigV1, error) {
-	rlk, err := lock.RLockedOpenFile(pathJoin(fsPath, minioMetaBucket, fsFormatJSONFile))
+	rlk, err := lock.RLockedOpenFile(preparePath(fsFormatPath))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, errUnformattedDisk
+			// If format.json not found then
+			// its an unformatted disk.
+			return nil, traceError(errUnformattedDisk)
 		}
-		return nil, err
+		return nil, traceError(err)
 	}
-	defer rlk.Close()
 
-	formatBytes, err := ioutil.ReadAll(rlk)
-	if err != nil {
+	var format = &formatConfigV1{}
+	if err = format.LoadFormat(rlk.LockedFile); err != nil {
+		rlk.Close()
 		return nil, err
 	}
 
-	format := &formatConfigV1{}
-	if err = json.Unmarshal(formatBytes, format); err != nil {
+	// Check format FS.
+	if err = format.CheckFS(); err != nil {
+		rlk.Close()
 		return nil, err
 	}
 
-	return format, nil
+	//  Always return read lock here and should be closed by the caller.
+	return rlk, traceError(err)
 }
 
-// writes FS format (format.json) into minioMetaBucket.
-func saveFormatFS(formatPath string, fsFormat *formatConfigV1) error {
-	metadataBytes, err := json.Marshal(fsFormat)
-	if err != nil {
-		return err
-	}
+// Creates a new format.json if unformatted.
+func createFormatFS(fsPath string) error {
+	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
 
-	// fsFormatJSONFile - format.json file stored in minioMetaBucket(.minio.sys) directory.
-	lk, err := lock.LockedOpenFile(preparePath(formatPath), os.O_CREATE|os.O_WRONLY, 0600)
+	// Attempt a write lock on formatConfigFile `format.json`
+	// file stored in minioMetaBucket(.minio.sys) directory.
+	lk, err := lock.TryLockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
-		return err
+		return traceError(err)
 	}
+	// Close the locked file upon return.
 	defer lk.Close()
 
-	_, err = lk.Write(metadataBytes)
-	// Success.
+	// Load format on disk, checks if we are unformatted
+	// writes the new format.json
+	var format = &formatConfigV1{}
+	err = format.LoadFormat(lk)
+	if errorCause(err) == errUnformattedDisk {
+		_, err = newFSFormat().WriteTo(lk)
+		return err
+	}
 	return err
 }
 
+func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) {
+	// This loop validates format.json by holding a read lock and
+	// proceeds if disk unformatted to hold non-blocking WriteLock
+	// If for some reason non-blocking WriteLock fails and the error
+	// is lock.ErrAlreadyLocked i.e some other process is holding a
+	// lock we retry in the loop again.
+	for {
+		// Validate the `format.json` for expected values.
+		rlk, err = checkLockedValidFormatFS(fsPath)
+		switch {
+		case err == nil:
+			// Holding a read lock ensures that any write lock operation
+			// is blocked if attempted in-turn avoiding corruption on
+			// the backend disk.
+			return rlk, nil
+		case errorCause(err) == errUnformattedDisk:
+			if err = createFormatFS(fsPath); err != nil {
+				// Existing write locks detected.
+				if errorCause(err) == lock.ErrAlreadyLocked {
+					// Lock already present, sleep and attempt again.
+					time.Sleep(100 * time.Millisecond)
+					continue
+				}
+
+				// Unexpected error, return.
+				return nil, err
+			}
+
+			// Loop will continue to attempt a read-lock on `format.json`.
+		default:
+			// Unhandled error return.
+			return nil, err
+		}
+	}
+}
+
 // Return if the part info in uploadedParts and completeParts are same.
 func isPartsSame(uploadedParts []objectPartInfo, completeParts []completePart) bool {
 	if len(uploadedParts) != len(completeParts) {

cmd/fs-v1-metadata_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"os"
 	"path/filepath"
 	"testing"
 )
@@ -49,7 +48,7 @@ func TestReadFSMetadata(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 	sha256sum := ""
@@ -59,7 +58,7 @@ func TestReadFSMetadata(t *testing.T) {
 	}
 
 	// Construct the full path of fs.json
-	fsPath := pathJoin("buckets", bucketName, objectName, "fs.json")
+	fsPath := pathJoin(bucketMetaPrefix, bucketName, objectName, "fs.json")
 	fsPath = pathJoin(fs.fsPath, minioMetaBucket, fsPath)
 
 	rlk, err := fs.rwPool.Open(fsPath)
@@ -73,18 +72,6 @@ func TestReadFSMetadata(t *testing.T) {
 	if _, err = fsMeta.ReadFrom(rlk.LockedFile); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
-
-	// Corrupted fs.json
-	file, err := os.OpenFile(preparePath(fsPath), os.O_APPEND|os.O_WRONLY, 0666)
-	if err != nil {
-		t.Fatal("Unexpected error ", err)
-	}
-	file.Write([]byte{'a'})
-	file.Close()
-	fsMeta = fsMetaV1{}
-	if _, err := fsMeta.ReadFrom(rlk.LockedFile); err == nil {
-		t.Fatal("Should fail", err)
-	}
 }
 
 // TestWriteFSMetadata - tests of writeFSMetadata with healthy disk.
@@ -98,7 +85,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 	sha256sum := ""
@@ -108,7 +95,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	}
 
 	// Construct the full path of fs.json
-	fsPath := pathJoin("buckets", bucketName, objectName, "fs.json")
+	fsPath := pathJoin(bucketMetaPrefix, bucketName, objectName, "fs.json")
 	fsPath = pathJoin(fs.fsPath, minioMetaBucket, fsPath)
 
 	rlk, err := fs.rwPool.Open(fsPath)
@@ -123,10 +110,10 @@ func TestWriteFSMetadata(t *testing.T) {
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
-	if fsMeta.Version != "1.0.0" {
+	if fsMeta.Version != fsMetaVersion {
 		t.Fatalf("Unexpected version %s", fsMeta.Version)
 	}
-	if fsMeta.Format != "fs" {
+	if fsMeta.Format != fsMetaFormat {
 		t.Fatalf("Unexpected format %s", fsMeta.Format)
 	}
 }

cmd/fs-v1-multipart.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,6 @@ import (
 	"io"
 	"os"
 	pathutil "path"
-	"runtime"
 	"strings"
 	"time"
 
@@ -32,6 +31,11 @@ import (
 	"github.com/minio/sha256-simd"
 )
 
+const (
+	fsMultipartExpiry        = time.Hour * 24 * 14
+	fsMultipartCleanupPeriod = time.Hour * 24
+)
+
 // Returns if the prefix is a multipart upload.
 func (fs fsObjects) isMultipartUpload(bucket, prefix string) bool {
 	uploadsIDPath := pathJoin(fs.fsPath, bucket, prefix, uploadsJSONFile)
@@ -46,56 +50,15 @@ func (fs fsObjects) isMultipartUpload(bucket, prefix string) bool {
 	return true
 }
 
-// Delete uploads.json file wrapper handling a tricky case on windows.
+// Delete uploads.json file wrapper
 func (fs fsObjects) deleteUploadsJSON(bucket, object, uploadID string) error {
-	timeID := fmt.Sprintf("%X", time.Now().UTC().UnixNano())
-	tmpPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID+"+"+timeID)
-
 	multipartBucketPath := pathJoin(fs.fsPath, minioMetaMultipartBucket)
 	uploadPath := pathJoin(multipartBucketPath, bucket, object)
 	uploadsMetaPath := pathJoin(uploadPath, uploadsJSONFile)
 
-	// Special case for windows please read through.
-	if runtime.GOOS == globalWindowsOSName {
-		// Ordinarily windows does not permit deletion or renaming of files still
-		// in use, but if all open handles to that file were opened with FILE_SHARE_DELETE
-		// then it can permit renames and deletions of open files.
-		//
-		// There are however some gotchas with this, and it is worth listing them here.
-		// Firstly, Windows never allows you to really delete an open file, rather it is
-		// flagged as delete pending and its entry in its directory remains visible
-		// (though no new file handles may be opened to it) and when the very last
-		// open handle to the file in the system is closed, only then is it truly
-		// deleted. Well, actually only sort of truly deleted, because Windows only
-		// appears to remove the file entry from the directory, but in fact that
-		// entry is merely hidden and actually still exists and attempting to create
-		// a file with the same name will return an access denied error. How long it
-		// silently exists for depends on a range of factors, but put it this way:
-		// if your code loops creating and deleting the same file name as you might
-		// when operating a lock file, you're going to see lots of random spurious
-		// access denied errors and truly dismal lock file performance compared to POSIX.
-		//
-		// We work-around these un-POSIX file semantics by taking a dual step to
-		// deleting files. Firstly, it renames the file to tmp location into multipartTmpBucket
-		// We always open files with FILE_SHARE_DELETE permission enabled, with that
-		// flag Windows permits renaming and deletion, and because the name was changed
-		// to a very random name somewhere not in its origin directory before deletion,
-		// you don't see those unexpected random errors when creating files with the
-		// same name as a recently deleted file as you do anywhere else on Windows.
-		// Because the file is probably not in its original containing directory any more,
-		// deletions of that directory will not fail with "directory not empty" as they
-		// otherwise normally would either.
-		fsRenameFile(uploadsMetaPath, tmpPath)
+	tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)
 
-		// Proceed to deleting the directory.
-		if err := fsDeleteFile(multipartBucketPath, uploadPath); err != nil {
-			return err
-		}
-
-		// Finally delete the renamed file.
-		return fsDeleteFile(pathutil.Dir(tmpPath), tmpPath)
-	}
-	return fsDeleteFile(multipartBucketPath, uploadsMetaPath)
+	return fsRemoveMeta(multipartBucketPath, uploadsMetaPath, tmpDir)
 }
 
 // Removes the uploadID, called either by CompleteMultipart of AbortMultipart. If the resuling uploads
@@ -201,7 +164,7 @@ func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarke
 }
 
 // listMultipartUploads - lists all multipart uploads.
-func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
 	result := ListMultipartsInfo{}
 	recursive := true
 	if delimiter == slashSeparator {
@@ -233,7 +196,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 	if uploadIDMarker != "" {
 		uploads, _, err = fs.listMultipartUploadIDs(bucket, keyMarker, uploadIDMarker, maxUploads)
 		if err != nil {
-			return ListMultipartsInfo{}, err
+			return lmi, err
 		}
 		maxUploads = maxUploads - len(uploads)
 	}
@@ -274,7 +237,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 					eof = true
 					break
 				}
-				return ListMultipartsInfo{}, walkResult.err
+				return lmi, walkResult.err
 			}
 
 			entry := strings.TrimPrefix(walkResult.entry, retainSlash(bucket))
@@ -298,7 +261,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 
 			tmpUploads, end, err = fs.listMultipartUploadIDs(bucket, entry, uploadIDMarker, maxUploads)
 			if err != nil {
-				return ListMultipartsInfo{}, err
+				return lmi, err
 			}
 
 			uploads = append(uploads, tmpUploads...)
@@ -344,22 +307,41 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 	return result, nil
 }
 
-// ListMultipartUploads - lists all the pending multipart uploads on a
+// ListMultipartUploads - returns empty response always. The onus is
+// on applications to remember uploadId of the multipart uploads that
+// are in progress.
+func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+	if err := checkListMultipartArgs(bucket, prefix, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
+		return lmi, err
+	}
+
+	if _, err := fs.statBucketDir(bucket); err != nil {
+		return lmi, toObjectErr(err, bucket)
+	}
+	return ListMultipartsInfo{
+		Prefix:         prefix,
+		KeyMarker:      keyMarker,
+		UploadIDMarker: uploadIDMarker,
+		Delimiter:      delimiter,
+		MaxUploads:     maxUploads,
+		IsTruncated:    false,
+	}, nil
+}
+
+// listMultipartUploadsHelper - lists all the pending multipart uploads on a
 // bucket. Additionally takes 'prefix, keyMarker, uploadIDmarker and a
 // delimiter' which allows us to list uploads match a particular
 // prefix or lexically starting from 'keyMarker' or delimiting the
 // output to get a directory like listing.
 //
-// Implements S3 compatible ListMultipartUploads API. The resulting
-// ListMultipartsInfo structure is unmarshalled directly into XML and
-// replied back to the client.
-func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+// This function remains here to aid in listing uploads that require cleanup.
+func (fs fsObjects) listMultipartUploadsHelper(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
 	if err := checkListMultipartArgs(bucket, prefix, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
-		return ListMultipartsInfo{}, err
+		return lmi, err
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return ListMultipartsInfo{}, toObjectErr(err, bucket)
+		return lmi, toObjectErr(err, bucket)
 	}
 
 	return fs.listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
@@ -380,7 +362,7 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st
 	fsMeta.Meta = meta
 
 	uploadID = mustGetUUID()
-	initiated := time.Now().UTC()
+	initiated := UTCNow()
 
 	// Add upload ID to uploads.json
 	uploadsPath := pathJoin(bucket, object, uploadsJSONFile)
@@ -454,9 +436,9 @@ func partToAppend(fsMeta fsMetaV1, fsAppendMeta fsMetaV1) (part objectPartInfo,
 // CopyObjectPart - similar to PutObjectPart but reads data from an existing
 // object. Internally incoming data is written to '.minio.sys/tmp' location
 // and safely renamed to '.minio.sys/multipart' for reach parts.
-func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64) (PartInfo, error) {
+func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64) (pi PartInfo, e error) {
 	if err := checkNewMultipartArgs(srcBucket, srcObject, fs); err != nil {
-		return PartInfo{}, err
+		return pi, err
 	}
 
 	// Initialize pipe.
@@ -473,7 +455,7 @@ func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, u
 
 	partInfo, err := fs.PutObjectPart(dstBucket, dstObject, uploadID, partID, length, pipeReader, "", "")
 	if err != nil {
-		return PartInfo{}, toObjectErr(err, dstBucket, dstObject)
+		return pi, toObjectErr(err, dstBucket, dstObject)
 	}
 
 	// Explicitly close the reader.
@@ -486,13 +468,13 @@ func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, u
 // an ongoing multipart transaction. Internally incoming data is
 // written to '.minio.sys/tmp' location and safely renamed to
 // '.minio.sys/multipart' for reach parts.
-func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (PartInfo, error) {
+func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (pi PartInfo, e error) {
 	if err := checkPutObjectPartArgs(bucket, object, fs); err != nil {
-		return PartInfo{}, err
+		return pi, err
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return PartInfo{}, toObjectErr(err, bucket)
+		return pi, toObjectErr(err, bucket)
 	}
 
 	// Hold the lock so that two parallel complete-multipart-uploads
@@ -505,9 +487,9 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	uploadsPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, object, uploadsJSONFile)
 	if _, err := fs.rwPool.Open(uploadsPath); err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return PartInfo{}, traceError(InvalidUploadID{UploadID: uploadID})
+			return pi, traceError(InvalidUploadID{UploadID: uploadID})
 		}
-		return PartInfo{}, toObjectErr(traceError(err), bucket, object)
+		return pi, toObjectErr(traceError(err), bucket, object)
 	}
 	defer fs.rwPool.Close(uploadsPath)
 
@@ -518,16 +500,16 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	rwlk, err := fs.rwPool.Write(fsMetaPath)
 	if err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return PartInfo{}, traceError(InvalidUploadID{UploadID: uploadID})
+			return pi, traceError(InvalidUploadID{UploadID: uploadID})
 		}
-		return PartInfo{}, toObjectErr(traceError(err), bucket, object)
+		return pi, toObjectErr(traceError(err), bucket, object)
 	}
 	defer rwlk.Close()
 
 	fsMeta := fsMetaV1{}
 	_, err = fsMeta.ReadFrom(rwlk)
 	if err != nil {
-		return PartInfo{}, toObjectErr(err, minioMetaMultipartBucket, fsMetaPath)
+		return pi, toObjectErr(err, minioMetaMultipartBucket, fsMetaPath)
 	}
 
 	partSuffix := fmt.Sprintf("object%d", partID)
@@ -565,14 +547,14 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	bytesWritten, cErr := fsCreateFile(fsPartPath, teeReader, buf, size)
 	if cErr != nil {
 		fsRemoveFile(fsPartPath)
-		return PartInfo{}, toObjectErr(cErr, minioMetaTmpBucket, tmpPartPath)
+		return pi, toObjectErr(cErr, minioMetaTmpBucket, tmpPartPath)
 	}
 
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.
 	if bytesWritten < size {
 		fsRemoveFile(fsPartPath)
-		return PartInfo{}, traceError(IncompleteBody{})
+		return pi, traceError(IncompleteBody{})
 	}
 
 	// Delete temporary part in case of failure. If
@@ -583,14 +565,14 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
 	if md5Hex != "" {
 		if newMD5Hex != md5Hex {
-			return PartInfo{}, traceError(BadDigest{md5Hex, newMD5Hex})
+			return pi, traceError(BadDigest{md5Hex, newMD5Hex})
 		}
 	}
 
 	if sha256sum != "" {
 		newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
 		if newSHA256sum != sha256sum {
-			return PartInfo{}, traceError(SHA256Mismatch{})
+			return pi, traceError(SHA256Mismatch{})
 		}
 	}
 
@@ -603,20 +585,20 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	fsNSPartPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, partPath)
 	if err = fsRenameFile(fsPartPath, fsNSPartPath); err != nil {
 		partLock.Unlock()
-		return PartInfo{}, toObjectErr(err, minioMetaMultipartBucket, partPath)
+		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
 	// Save the object part info in `fs.json`.
 	fsMeta.AddObjectPart(partID, partSuffix, newMD5Hex, size)
 	if _, err = fsMeta.WriteTo(rwlk); err != nil {
 		partLock.Unlock()
-		return PartInfo{}, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
+		return pi, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
 	}
 
 	partNamePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, partSuffix)
 	fi, err := fsStatFile(partNamePath)
 	if err != nil {
-		return PartInfo{}, toObjectErr(err, minioMetaMultipartBucket, partSuffix)
+		return pi, toObjectErr(err, minioMetaMultipartBucket, partSuffix)
 	}
 
 	// Append the part in background.
@@ -638,103 +620,37 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	}, nil
 }
 
-// listObjectParts - wrapper scanning through
-// '.minio.sys/multipart/bucket/object/UPLOADID'. Lists all the parts
-// saved inside '.minio.sys/multipart/bucket/object/UPLOADID'.
-func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) {
-	result := ListPartsInfo{}
-
-	uploadIDPath := pathJoin(bucket, object, uploadID)
-	fsMetaPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, fsMetaJSONFile)
-	metaFile, err := fs.rwPool.Open(fsMetaPath)
-	if err != nil {
-		if err == errFileNotFound || err == errFileAccessDenied {
-			// On windows oddly this is returned.
-			return ListPartsInfo{}, traceError(InvalidUploadID{UploadID: uploadID})
-		}
-		return ListPartsInfo{}, toObjectErr(traceError(err), bucket, object)
-	}
-	defer fs.rwPool.Close(fsMetaPath)
-
-	fsMeta := fsMetaV1{}
-	_, err = fsMeta.ReadFrom(metaFile.LockedFile)
-	if err != nil {
-		return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, fsMetaPath)
-	}
-
-	// Only parts with higher part numbers will be listed.
-	partIdx := fsMeta.ObjectPartIndex(partNumberMarker)
-	parts := fsMeta.Parts
-	if partIdx != -1 {
-		parts = fsMeta.Parts[partIdx+1:]
-	}
-
-	count := maxParts
-	for _, part := range parts {
-		var fi os.FileInfo
-		partNamePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, part.Name)
-		fi, err = fsStatFile(partNamePath)
-		if err != nil {
-			return ListPartsInfo{}, toObjectErr(err, minioMetaMultipartBucket, partNamePath)
-		}
-		result.Parts = append(result.Parts, PartInfo{
-			PartNumber:   part.Number,
-			ETag:         part.ETag,
-			LastModified: fi.ModTime(),
-			Size:         fi.Size(),
-		})
-		count--
-		if count == 0 {
-			break
-		}
-	}
-
-	// If listed entries are more than maxParts, we set IsTruncated as true.
-	if len(parts) > len(result.Parts) {
-		result.IsTruncated = true
-		// Make sure to fill next part number marker if IsTruncated is
-		// true for subsequent listing.
-		nextPartNumberMarker := result.Parts[len(result.Parts)-1].PartNumber
-		result.NextPartNumberMarker = nextPartNumberMarker
-	}
-	result.Bucket = bucket
-	result.Object = object
-	result.UploadID = uploadID
-	result.MaxParts = maxParts
-
-	// Success.
-	return result, nil
-}
-
-// ListObjectParts - lists all previously uploaded parts for a given
-// object and uploadID.  Takes additional input of part-number-marker
-// to indicate where the listing should begin from.
-//
-// Implements S3 compatible ListObjectParts API. The resulting
-// ListPartsInfo structure is unmarshalled directly into XML and
-// replied back to the client.
-func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) {
+// ListObjectParts - returns empty response always. Applications are
+// expected to remember the uploaded part numbers and corresponding
+// etags.
+func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi ListPartsInfo, e error) {
 	if err := checkListPartsArgs(bucket, object, fs); err != nil {
-		return ListPartsInfo{}, err
+		return lpi, err
 	}
 
+	// Check if bucket exists
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return ListPartsInfo{}, toObjectErr(err, bucket)
+		return lpi, toObjectErr(err, bucket)
 	}
 
-	// Hold the lock so that two parallel complete-multipart-uploads
-	// do not leave a stale uploads.json behind.
-	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
-	objectMPartPathLock.RLock()
-	defer objectMPartPathLock.RUnlock()
-
-	listPartsInfo, err := fs.listObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
+	// Check if uploadID exists. N B This check may race with a
+	// concurrent abortMultipartUpload on the same uploadID. It is
+	// OK to be eventually consistent w.r.t listing of objects,
+	// uploads and parts.
+	uploadIDPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, object, uploadID)
+	_, err := fsStatDir(uploadIDPath)
 	if err != nil {
-		return ListPartsInfo{}, toObjectErr(err, bucket, object)
+		return lpi, traceError(InvalidUploadID{UploadID: uploadID})
 	}
 
-	// Success.
-	return listPartsInfo, nil
+	// Return empty list parts response
+	return ListPartsInfo{
+		Bucket:           bucket,
+		Object:           object,
+		UploadID:         uploadID,
+		PartNumberMarker: 0,
+		MaxParts:         maxParts,
+	}, nil
 }
 
 // CompleteMultipartUpload - completes an ongoing multipart
@@ -743,19 +659,24 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
 // md5sums of all the parts.
 //
 // Implements S3 compatible Complete multipart API.
-func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (ObjectInfo, error) {
+func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (oi ObjectInfo, e error) {
 	if err := checkCompleteMultipartArgs(bucket, object, fs); err != nil {
-		return ObjectInfo{}, err
+		return oi, err
+	}
+
+	// Check if an object is present as one of the parent dir.
+	if fs.parentDirIsObject(bucket, pathutil.Dir(object)) {
+		return oi, toObjectErr(traceError(errFileAccessDenied), bucket, object)
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket)
+		return oi, toObjectErr(err, bucket)
 	}
 
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5, err := getCompleteMultipartMD5(parts)
 	if err != nil {
-		return ObjectInfo{}, err
+		return oi, err
 	}
 
 	uploadIDPath := pathJoin(bucket, object, uploadID)
@@ -770,9 +691,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	rlk, err := fs.rwPool.Open(fsMetaPathMultipart)
 	if err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return ObjectInfo{}, traceError(InvalidUploadID{UploadID: uploadID})
+			return oi, traceError(InvalidUploadID{UploadID: uploadID})
 		}
-		return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(traceError(err), bucket, object)
 	}
 
 	// Disallow any parallel abort or complete multipart operations.
@@ -780,9 +701,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	if err != nil {
 		fs.rwPool.Close(fsMetaPathMultipart)
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return ObjectInfo{}, traceError(InvalidUploadID{UploadID: uploadID})
+			return oi, traceError(InvalidUploadID{UploadID: uploadID})
 		}
-		return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(traceError(err), bucket, object)
 	}
 	defer rwlk.Close()
 
@@ -791,7 +712,31 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	_, err = fsMeta.ReadFrom(rlk.LockedFile)
 	if err != nil {
 		fs.rwPool.Close(fsMetaPathMultipart)
-		return ObjectInfo{}, toObjectErr(err, minioMetaMultipartBucket, fsMetaPathMultipart)
+		return oi, toObjectErr(err, minioMetaMultipartBucket, fsMetaPathMultipart)
+	}
+
+	// Validate all parts and then commit to disk.
+	for i, part := range parts {
+		partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
+		if partIdx == -1 {
+			fs.rwPool.Close(fsMetaPathMultipart)
+			return oi, traceError(InvalidPart{})
+		}
+
+		if fsMeta.Parts[partIdx].ETag != part.ETag {
+			fs.rwPool.Close(fsMetaPathMultipart)
+			return oi, traceError(InvalidPart{})
+		}
+
+		// All parts except the last part has to be atleast 5MB.
+		if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
+			fs.rwPool.Close(fsMetaPathMultipart)
+			return oi, traceError(PartTooSmall{
+				PartNumber: part.PartNumber,
+				PartSize:   fsMeta.Parts[partIdx].Size,
+				PartETag:   part.ETag,
+			})
+		}
 	}
 
 	// Wait for any competing PutObject() operation on bucket/object, since same namespace
@@ -800,7 +745,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	metaFile, err := fs.rwPool.Create(fsMetaPath)
 	if err != nil {
 		fs.rwPool.Close(fsMetaPathMultipart)
-		return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(traceError(err), bucket, object)
 	}
 	defer metaFile.Close()
 
@@ -817,7 +762,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 			fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID)
 			if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil {
 				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, toObjectErr(err, minioMetaTmpBucket, uploadID)
+				return oi, toObjectErr(err, minioMetaTmpBucket, uploadID)
 			}
 		}
 	}
@@ -835,29 +780,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		// Allocate staging buffer.
 		var buf = make([]byte, readSizeV1)
 
-		// Validate all parts and then commit to disk.
-		for i, part := range parts {
-			partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
-			if partIdx == -1 {
-				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, traceError(InvalidPart{})
-			}
-
-			if fsMeta.Parts[partIdx].ETag != part.ETag {
-				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, traceError(BadDigest{})
-			}
-
-			// All parts except the last part has to be atleast 5MB.
-			if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
-				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, traceError(PartTooSmall{
-					PartNumber: part.PartNumber,
-					PartSize:   fsMeta.Parts[partIdx].Size,
-					PartETag:   part.ETag,
-				})
-			}
-
+		for _, part := range parts {
 			// Construct part suffix.
 			partSuffix := fmt.Sprintf("object%d", part.PartNumber)
 			multipartPartFile := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, partSuffix)
@@ -868,9 +791,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 			if err != nil {
 				fs.rwPool.Close(fsMetaPathMultipart)
 				if err == errFileNotFound {
-					return ObjectInfo{}, traceError(InvalidPart{})
+					return oi, traceError(InvalidPart{})
 				}
-				return ObjectInfo{}, toObjectErr(traceError(err), minioMetaMultipartBucket, partSuffix)
+				return oi, toObjectErr(traceError(err), minioMetaMultipartBucket, partSuffix)
 			}
 
 			// No need to hold a lock, this is a unique file and will be only written
@@ -880,7 +803,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 			if err != nil {
 				reader.Close()
 				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+				return oi, toObjectErr(traceError(err), bucket, object)
 			}
 
 			_, err = io.CopyBuffer(wfile, reader, buf)
@@ -888,7 +811,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 				wfile.Close()
 				reader.Close()
 				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+				return oi, toObjectErr(traceError(err), bucket, object)
 			}
 
 			wfile.Close()
@@ -897,7 +820,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 
 		if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil {
 			fs.rwPool.Close(fsMetaPathMultipart)
-			return ObjectInfo{}, toObjectErr(err, minioMetaTmpBucket, uploadID)
+			return oi, toObjectErr(err, minioMetaTmpBucket, uploadID)
 		}
 	}
 
@@ -908,12 +831,12 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	if len(fsMeta.Meta) == 0 {
 		fsMeta.Meta = make(map[string]string)
 	}
-	fsMeta.Meta["md5Sum"] = s3MD5
+	fsMeta.Meta["etag"] = s3MD5
 
 	// Write all the set metadata.
 	if _, err = fsMeta.WriteTo(metaFile); err != nil {
 		fs.rwPool.Close(fsMetaPathMultipart)
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
+		return oi, toObjectErr(err, bucket, object)
 	}
 
 	// Close lock held on bucket/object/uploadid/fs.json,
@@ -925,17 +848,17 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	multipartObjectDir := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, object)
 	multipartUploadIDDir := pathJoin(multipartObjectDir, uploadID)
 	if err = fsRemoveUploadIDPath(multipartObjectDir, multipartUploadIDDir); err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
+		return oi, toObjectErr(err, bucket, object)
 	}
 
 	// Remove entry from `uploads.json`.
 	if err = fs.removeUploadID(bucket, object, uploadID, rwlk); err != nil {
-		return ObjectInfo{}, toObjectErr(err, minioMetaMultipartBucket, pathutil.Join(bucket, object))
+		return oi, toObjectErr(err, minioMetaMultipartBucket, pathutil.Join(bucket, object))
 	}
 
 	fi, err := fsStatFile(fsNSObjPath)
 	if err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
+		return oi, toObjectErr(err, bucket, object)
 	}
 
 	// Return object info.
@@ -1013,3 +936,56 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 
 	return nil
 }
+
+// Remove multipart uploads left unattended in a given bucket older than `fsMultipartExpiry`
+func (fs fsObjects) cleanupStaleMultipartUpload(bucket string) (err error) {
+	var lmi ListMultipartsInfo
+
+	for {
+		// List multipart uploads in a bucket 1000 at a time
+		prefix := ""
+		lmi, err = fs.listMultipartUploadsHelper(bucket, prefix, lmi.KeyMarker, lmi.UploadIDMarker, slashSeparator, 1000)
+		if err != nil {
+			errorIf(err, fmt.Sprintf("Failed to list uploads of %s for cleaning up of multipart uploads older than %d weeks", bucket, fsMultipartExpiry))
+			return err
+		}
+
+		// Remove uploads (and its parts) older than 2 weeks
+		for _, upload := range lmi.Uploads {
+			uploadIDPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, upload.Object, upload.UploadID)
+			st, err := fsStatDir(uploadIDPath)
+			if err != nil {
+				errorIf(err, "Failed to stat uploads directory", uploadIDPath)
+				return err
+			}
+			if time.Since(st.ModTime()) > fsMultipartExpiry {
+				fs.AbortMultipartUpload(bucket, upload.Object, upload.UploadID)
+			}
+		}
+
+		// No more incomplete uploads remain
+		if !lmi.IsTruncated {
+			break
+		}
+	}
+
+	return nil
+}
+
+// Remove multipart uploads left unattended for more than `fsMultipartExpiry` seconds.
+func (fs fsObjects) cleanupStaleMultipartUploads() {
+	for {
+		bucketInfos, err := fs.ListBuckets()
+		if err != nil {
+			errorIf(err, fmt.Sprintf("Failed to list buckets for cleaning up of multipart uploads older than %d weeks", fsMultipartExpiry))
+			return
+		}
+
+		for _, bucketInfo := range bucketInfos {
+			fs.cleanupStaleMultipartUpload(bucketInfo.Name)
+		}
+
+		// Schedule for the next multipart backend cleanup
+		time.Sleep(fsMultipartCleanupPeriod)
+	}
+}

cmd/fs-v1-multipart_test.go

@@ -33,7 +33,7 @@ func TestFSWriteUploadJSON(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	obj.MakeBucket(bucketName)
+	obj.MakeBucketWithLocation(bucketName, "")
 	_, err := obj.NewMultipartUpload(bucketName, objectName, nil)
 	if err != nil {
 		t.Fatal("Unexpected err: ", err)
@@ -60,7 +60,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -91,7 +91,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	data := []byte("12345")
 	dataLen := int64(len(data))
 
-	if err = obj.MakeBucket(bucketName); err != nil {
+	if err = obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -122,7 +122,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -161,7 +161,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 

cmd/fs-v1.go

@@ -22,7 +22,9 @@ import (
 	"fmt"
 	"hash"
 	"io"
+	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"sort"
 	"syscall"
@@ -40,6 +42,9 @@ type fsObjects struct {
 	// temporary transactions.
 	fsUUID string
 
+	// This value shouldn't be touched, once initialized.
+	fsFormatRlk *lock.RLockedFile // Is a read lock on `format.json`.
+
 	// FS rw pool.
 	rwPool *fsIOPool
 
@@ -84,7 +89,7 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		return nil, err
 	}
 
-	fi, err := os.Stat(preparePath(fsPath))
+	fi, err := osStat(preparePath(fsPath))
 	if err == nil {
 		if !fi.IsDir() {
 			return nil, syscall.ENOTDIR
@@ -98,6 +103,16 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		}
 	}
 
+	di, err := getDiskInfo(preparePath(fsPath))
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if disk has minimum required total space.
+	if err = checkDiskMinTotal(di); err != nil {
+		return nil, err
+	}
+
 	// Assign a new UUID for FS minio mode. Each server instance
 	// gets its own UUID for temporary file transaction.
 	fsUUID := mustGetUUID()
@@ -107,24 +122,10 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		return nil, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
 	}
 
-	// Load `format.json`.
-	format, err := loadFormatFS(fsPath)
-	if err != nil && err != errUnformattedDisk {
-		return nil, fmt.Errorf("Unable to load 'format.json', %s", err)
-	}
-
-	// If the `format.json` doesn't exist create one.
-	if err == errUnformattedDisk {
-		fsFormatPath := pathJoin(fsPath, minioMetaBucket, fsFormatJSONFile)
-		// Initialize format.json, if already exists overwrite it.
-		if serr := saveFormatFS(fsFormatPath, newFSFormatV1()); serr != nil {
-			return nil, fmt.Errorf("Unable to initialize 'format.json', %s", serr)
-		}
-	}
-
-	// Validate if we have the same format.
-	if err == nil && format.Format != "fs" {
-		return nil, fmt.Errorf("Unable to recognize backend format, Disk is not in FS format. %s", format.Format)
+	// Initialize `format.json`, this function also returns.
+	rlk, err := initFormatFS(fsPath)
+	if err != nil {
+		return nil, err
 	}
 
 	// Initialize fs objects.
@@ -140,18 +141,25 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		},
 	}
 
+	// Once the filesystem has initialized hold the read lock for
+	// the life time of the server. This is done to ensure that under
+	// shared backend mode for FS, remote servers do not migrate
+	// or cause changes on backend format.
+	fs.fsFormatRlk = rlk
+
 	// Initialize and load bucket policies.
-	err = initBucketPolicies(fs)
-	if err != nil {
+	if err = initBucketPolicies(fs); err != nil {
 		return nil, fmt.Errorf("Unable to load all bucket policies. %s", err)
 	}
 
 	// Initialize a new event notifier.
-	err = initEventNotifier(fs)
-	if err != nil {
+	if err = initEventNotifier(fs); err != nil {
 		return nil, fmt.Errorf("Unable to initialize event notification. %s", err)
 	}
 
+	// Start background process to cleanup old files in minio.sys.tmp
+	go fs.cleanupStaleMultipartUploads()
+
 	// Return successfully initialized object layer.
 	return fs, nil
 }
@@ -203,7 +211,7 @@ func (fs fsObjects) statBucketDir(bucket string) (os.FileInfo, error) {
 
 // MakeBucket - create a new bucket, returns if it
 // already exists.
-func (fs fsObjects) MakeBucket(bucket string) error {
+func (fs fsObjects) MakeBucketWithLocation(bucket, location string) error {
 	bucketDir, err := fs.getBucketDir(bucket)
 	if err != nil {
 		return toObjectErr(err, bucket)
@@ -217,13 +225,13 @@ func (fs fsObjects) MakeBucket(bucket string) error {
 }
 
 // GetBucketInfo - fetch bucket metadata info.
-func (fs fsObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
+func (fs fsObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
 	st, err := fs.statBucketDir(bucket)
 	if err != nil {
-		return BucketInfo{}, toObjectErr(err, bucket)
+		return bi, toObjectErr(err, bucket)
 	}
 
-	// As os.Stat() doesn't carry other than ModTime(), use ModTime() as CreatedTime.
+	// As osStat() doesn't carry other than ModTime(), use ModTime() as CreatedTime.
 	createdTime := st.ModTime()
 	return BucketInfo{
 		Name:    bucket,
@@ -262,7 +270,7 @@ func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
 
 		bucketInfos = append(bucketInfos, BucketInfo{
 			Name: fi.Name(),
-			// As os.Stat() doesnt carry CreatedTime, use ModTime() as CreatedTime.
+			// As osStat() doesnt carry CreatedTime, use ModTime() as CreatedTime.
 			Created: fi.ModTime(),
 		})
 	}
@@ -307,15 +315,15 @@ func (fs fsObjects) DeleteBucket(bucket string) error {
 // CopyObject - copy object source object to destination object.
 // if source object and destination object are same we only
 // update metadata.
-func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string) (ObjectInfo, error) {
+func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string) (oi ObjectInfo, e error) {
 	if _, err := fs.statBucketDir(srcBucket); err != nil {
-		return ObjectInfo{}, toObjectErr(err, srcBucket)
+		return oi, toObjectErr(err, srcBucket)
 	}
 
 	// Stat the file to get file size.
 	fi, err := fsStatFile(pathJoin(fs.fsPath, srcBucket, srcObject))
 	if err != nil {
-		return ObjectInfo{}, toObjectErr(err, srcBucket, srcObject)
+		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
 
 	// Check if this request is only metadata update.
@@ -325,7 +333,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 		var wlk *lock.LockedFile
 		wlk, err = fs.rwPool.Write(fsMetaPath)
 		if err != nil {
-			return ObjectInfo{}, toObjectErr(traceError(err), srcBucket, srcObject)
+			return oi, toObjectErr(traceError(err), srcBucket, srcObject)
 		}
 		// This close will allow for locks to be synchronized on `fs.json`.
 		defer wlk.Close()
@@ -334,7 +342,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 		fsMeta := newFSMetaV1()
 		fsMeta.Meta = metadata
 		if _, err = fsMeta.WriteTo(wlk); err != nil {
-			return ObjectInfo{}, toObjectErr(err, srcBucket, srcObject)
+			return oi, toObjectErr(err, srcBucket, srcObject)
 		}
 
 		// Return the new object info.
@@ -359,7 +367,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 
 	objInfo, err := fs.PutObject(dstBucket, dstObject, length, pipeReader, metadata, "")
 	if err != nil {
-		return ObjectInfo{}, toObjectErr(err, dstBucket, dstObject)
+		return oi, toObjectErr(err, dstBucket, dstObject)
 	}
 
 	// Explicitly close the reader.
@@ -434,7 +442,7 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
 }
 
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
-func (fs fsObjects) getObjectInfo(bucket, object string) (ObjectInfo, error) {
+func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
 	fsMeta := fsMetaV1{}
 	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
 
@@ -449,61 +457,87 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (ObjectInfo, error) {
 			// PutObject() transaction, if we arrive at such
 			// a situation we just ignore and continue.
 			if errorCause(rerr) != io.EOF {
-				return ObjectInfo{}, toObjectErr(rerr, bucket, object)
+				return oi, toObjectErr(rerr, bucket, object)
 			}
 		}
 	}
 
 	// Ignore if `fs.json` is not available, this is true for pre-existing data.
 	if err != nil && err != errFileNotFound {
-		return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(traceError(err), bucket, object)
 	}
 
 	// Stat the file to get file size.
 	fi, err := fsStatFile(pathJoin(fs.fsPath, bucket, object))
 	if err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
+		return oi, toObjectErr(err, bucket, object)
 	}
 
 	return fsMeta.ToObjectInfo(bucket, object, fi), nil
 }
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
-func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
-	// This is a special case with object whose name ends with
-	// a slash separator, we always return object not found here.
-	if hasSuffix(object, slashSeparator) {
-		return ObjectInfo{}, toObjectErr(traceError(errFileNotFound), bucket, object)
-	}
-
+func (fs fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
 	if err := checkGetObjArgs(bucket, object); err != nil {
-		return ObjectInfo{}, err
+		return oi, err
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket)
+		return oi, toObjectErr(err, bucket)
 	}
 
 	return fs.getObjectInfo(bucket, object)
 }
 
+// This function does the following check, suppose
+// object is "a/b/c/d", stat makes sure that objects ""a/b/c""
+// "a/b" and "a" do not exist.
+func (fs fsObjects) parentDirIsObject(bucket, parent string) bool {
+	var isParentDirObject func(string) bool
+	isParentDirObject = func(p string) bool {
+		if p == "." {
+			return false
+		}
+		if _, err := fsStatFile(pathJoin(fs.fsPath, bucket, p)); err == nil {
+			// If there is already a file at prefix "p" return error.
+			return true
+		}
+		// Check if there is a file as one of the parent paths.
+		return isParentDirObject(path.Dir(p))
+	}
+	return isParentDirObject(parent)
+}
+
 // PutObject - creates an object upon reading from the input stream
 // until EOF, writes data directly to configured filesystem path.
 // Additionally writes `fs.json` which carries the necessary metadata
 // for future object operations.
-func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
-	// This is a special case with size as '0' and object ends with
-	// a slash separator, we treat it like a valid operation and
-	// return success.
+func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, retErr error) {
+	var err error
+
+	// Validate if bucket name is valid and exists.
+	if _, err = fs.statBucketDir(bucket); err != nil {
+		return ObjectInfo{}, toObjectErr(err, bucket)
+	}
+
+	// This is a special case with size as '0' and object ends
+	// with a slash separator, we treat it like a valid operation
+	// and return success.
 	if isObjectDir(object, size) {
+		// Check if an object is present as one of the parent dir.
+		if fs.parentDirIsObject(bucket, path.Dir(object)) {
+			return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
+		}
 		return dirObjectInfo(bucket, object, size, metadata), nil
 	}
+
 	if err = checkPutObjectArgs(bucket, object, fs); err != nil {
 		return ObjectInfo{}, err
 	}
 
-	if _, err = fs.statBucketDir(bucket); err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket)
+	// Check if an object is present as one of the parent dir.
+	if fs.parentDirIsObject(bucket, path.Dir(object)) {
+		return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
 	}
 
 	// No metadata is set, allocate a new one.
@@ -516,13 +550,21 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 
 	var wlk *lock.LockedFile
 	if bucket != minioMetaBucket {
-		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
+		bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix)
+		fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fsMetaJSONFile)
 		wlk, err = fs.rwPool.Create(fsMetaPath)
 		if err != nil {
 			return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
 		}
 		// This close will allow for locks to be synchronized on `fs.json`.
 		defer wlk.Close()
+		defer func() {
+			// Remove meta file when PutObject encounters any error
+			if retErr != nil {
+				tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)
+				fsRemoveMeta(bucketMetaDir, fsMetaPath, tmpDir)
+			}
+		}()
 	}
 
 	// Uploaded object will first be written to the temporary location which will eventually
@@ -581,12 +623,12 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 
 	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
 	// Update the md5sum if not set with the newly calculated one.
-	if len(metadata["md5Sum"]) == 0 {
-		metadata["md5Sum"] = newMD5Hex
+	if len(metadata["etag"]) == 0 {
+		metadata["etag"] = newMD5Hex
 	}
 
 	// md5Hex representation.
-	md5Hex := metadata["md5Sum"]
+	md5Hex := metadata["etag"]
 	if md5Hex != "" {
 		if newMD5Hex != md5Hex {
 			// Returns md5 mismatch.
@@ -687,20 +729,59 @@ func (fs fsObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
 	return listDir
 }
 
+// getObjectETag is a helper function, which returns only the md5sum
+// of the file on the disk.
+func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
+	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, entry, fsMetaJSONFile)
+
+	// Read `fs.json` to perhaps contend with
+	// parallel Put() operations.
+	rlk, err := fs.rwPool.Open(fsMetaPath)
+	// Ignore if `fs.json` is not available, this is true for pre-existing data.
+	if err != nil && err != errFileNotFound {
+		return "", toObjectErr(traceError(err), bucket, entry)
+	}
+
+	// If file is not found, we don't need to proceed forward.
+	if err == errFileNotFound {
+		return "", nil
+	}
+
+	// Read from fs metadata only if it exists.
+	defer fs.rwPool.Close(fsMetaPath)
+
+	fsMetaBuf, err := ioutil.ReadAll(rlk.LockedFile)
+	if err != nil {
+		// `fs.json` can be empty due to previously failed
+		// PutObject() transaction, if we arrive at such
+		// a situation we just ignore and continue.
+		if errorCause(err) != io.EOF {
+			return "", toObjectErr(err, bucket, entry)
+		}
+	}
+
+	// Check if FS metadata is valid, if not return error.
+	if !isFSMetaValid(parseFSVersion(fsMetaBuf), parseFSFormat(fsMetaBuf)) {
+		return "", toObjectErr(traceError(errCorruptedFormat), bucket, entry)
+	}
+
+	return extractETag(parseFSMetaMap(fsMetaBuf)), nil
+}
+
 // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
 // state for future re-entrant list requests.
-func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
 	if err := checkListObjsArgs(bucket, prefix, marker, delimiter, fs); err != nil {
-		return ListObjectsInfo{}, err
+		return loi, err
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return ListObjectsInfo{}, err
+		return loi, err
 	}
 
 	// With max keys of zero we have reached eof, return right here.
 	if maxKeys == 0 {
-		return ListObjectsInfo{}, nil
+		return loi, nil
 	}
 
 	// For delimiter and prefix as '/' we do not list anything at all
@@ -709,7 +790,7 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 	// as '/' we don't have any entries, since all the keys are
 	// of form 'keyName/...'
 	if delimiter == slashSeparator && prefix == slashSeparator {
-		return ListObjectsInfo{}, nil
+		return loi, nil
 	}
 
 	// Over flowing count - reset to maxObjectList.
@@ -731,14 +812,33 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 			objInfo.IsDir = true
 			return
 		}
+
+		// Protect reading `fs.json`.
+		objectLock := globalNSMutex.NewNSLock(bucket, entry)
+		objectLock.RLock()
+		var etag string
+		etag, err = fs.getObjectETag(bucket, entry)
+		objectLock.RUnlock()
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+
 		// Stat the file to get file size.
 		var fi os.FileInfo
 		fi, err = fsStatFile(pathJoin(fs.fsPath, bucket, entry))
 		if err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, entry)
 		}
-		fsMeta := fsMetaV1{}
-		return fsMeta.ToObjectInfo(bucket, entry, fi), nil
+
+		// Success.
+		return ObjectInfo{
+			Name:    entry,
+			Bucket:  bucket,
+			Size:    fi.Size(),
+			ModTime: fi.ModTime(),
+			IsDir:   fi.IsDir(),
+			ETag:    etag,
+		}, nil
 	}
 
 	heal := false // true only for xl.ListObjectsHeal()
@@ -771,13 +871,13 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 		if walkResult.err != nil {
 			// File not found is a valid case.
 			if errorCause(walkResult.err) == errFileNotFound {
-				return ListObjectsInfo{}, nil
+				return loi, nil
 			}
-			return ListObjectsInfo{}, toObjectErr(walkResult.err, bucket, prefix)
+			return loi, toObjectErr(walkResult.err, bucket, prefix)
 		}
 		objInfo, err := entryToObjectInfo(walkResult.entry)
 		if err != nil {
-			return ListObjectsInfo{}, nil
+			return loi, nil
 		}
 		nextMarker = objInfo.Name
 		objInfos = append(objInfos, objInfo)
@@ -809,8 +909,8 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 }
 
 // HealObject - no-op for fs. Valid only for XL.
-func (fs fsObjects) HealObject(bucket, object string) error {
-	return traceError(NotImplemented{})
+func (fs fsObjects) HealObject(bucket, object string) (int, int, error) {
+	return 0, 0, traceError(NotImplemented{})
 }
 
 // HealBucket - no-op for fs, Valid only for XL.
@@ -819,8 +919,8 @@ func (fs fsObjects) HealBucket(bucket string) error {
 }
 
 // ListObjectsHeal - list all objects to be healed. Valid only for XL
-func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
-	return ListObjectsInfo{}, traceError(NotImplemented{})
+func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
+	return loi, traceError(NotImplemented{})
 }
 
 // ListBucketsHeal - list all buckets to be healed. Valid only for XL
@@ -829,6 +929,6 @@ func (fs fsObjects) ListBucketsHeal() ([]BucketInfo, error) {
 }
 
 func (fs fsObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
-	delimiter string, maxUploads int) (ListMultipartsInfo, error) {
-	return ListMultipartsInfo{}, traceError(NotImplemented{})
+	delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+	return lmi, traceError(NotImplemented{})
 }

cmd/fs-v1_test.go

@@ -63,7 +63,7 @@ func TestFSShutdown(t *testing.T) {
 		obj := initFSObjects(disk, t)
 		fs := obj.(*fsObjects)
 		objectContent := "12345"
-		obj.MakeBucket(bucketName)
+		obj.MakeBucketWithLocation(bucketName, "")
 		sha256sum := ""
 		obj.PutObject(bucketName, objectName, int64(len(objectContent)), bytes.NewReader([]byte(objectContent)), nil, sha256sum)
 		return fs, disk
@@ -85,47 +85,6 @@ func TestFSShutdown(t *testing.T) {
 	}
 }
 
-// TestFSLoadFormatFS - test loadFormatFS with healty and faulty disks
-func TestFSLoadFormatFS(t *testing.T) {
-	// Prepare for testing
-	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
-
-	// Assign a new UUID.
-	uuid := mustGetUUID()
-
-	// Initialize meta volume, if volume already exists ignores it.
-	if err := initMetaVolumeFS(disk, uuid); err != nil {
-		t.Fatal(err)
-	}
-
-	fsFormatPath := pathJoin(disk, minioMetaBucket, fsFormatJSONFile)
-	if err := saveFormatFS(preparePath(fsFormatPath), newFSFormatV1()); err != nil {
-		t.Fatal("Should not fail here", err)
-	}
-	_, err := loadFormatFS(disk)
-	if err != nil {
-		t.Fatal("Should not fail here", err)
-	}
-	// Loading corrupted format file
-	file, err := os.OpenFile(preparePath(fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
-	if err != nil {
-		t.Fatal("Should not fail here", err)
-	}
-	file.Write([]byte{'b'})
-	file.Close()
-	_, err = loadFormatFS(disk)
-	if err == nil {
-		t.Fatal("Should return an error here")
-	}
-	// Loading format file from disk not found.
-	removeAll(disk)
-	_, err = loadFormatFS(disk)
-	if err != nil && err != errUnformattedDisk {
-		t.Fatal("Should return unformatted disk, but got", err)
-	}
-}
-
 // TestFSGetBucketInfo - test GetBucketInfo with healty and faulty disks
 func TestFSGetBucketInfo(t *testing.T) {
 	// Prepare for testing
@@ -136,7 +95,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 	fs := obj.(*fsObjects)
 	bucketName := "bucket"
 
-	obj.MakeBucket(bucketName)
+	obj.MakeBucketWithLocation(bucketName, "")
 
 	// Test with valid parameters
 	info, err := fs.GetBucketInfo(bucketName)
@@ -161,6 +120,74 @@ func TestFSGetBucketInfo(t *testing.T) {
 	}
 }
 
+func TestFSPutObject(t *testing.T) {
+	// Prepare for tests
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer removeAll(disk)
+
+	obj := initFSObjects(disk, t)
+	bucketName := "bucket"
+	objectName := "1/2/3/4/object"
+
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
+		t.Fatal(err)
+	}
+
+	sha256sum := ""
+
+	// With a regular object.
+	_, err := obj.PutObject(bucketName+"non-existent", objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	if err == nil {
+		t.Fatal("Unexpected should fail here, bucket doesn't exist")
+	}
+	if _, ok := errorCause(err).(BucketNotFound); !ok {
+		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
+	}
+
+	// With a directory object.
+	_, err = obj.PutObject(bucketName+"non-existent", objectName+"/", int64(0), bytes.NewReader([]byte("")), nil, sha256sum)
+	if err == nil {
+		t.Fatal("Unexpected should fail here, bucket doesn't exist")
+	}
+	if _, ok := errorCause(err).(BucketNotFound); !ok {
+		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
+	}
+
+	_, err = obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = obj.PutObject(bucketName, objectName+"/1", int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	if err == nil {
+		t.Fatal("Unexpected should fail here, backend corruption occurred")
+	}
+	if nerr, ok := errorCause(err).(PrefixAccessDenied); !ok {
+		t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
+	} else {
+		if nerr.Bucket != "bucket" {
+			t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
+		}
+		if nerr.Object != "1/2/3/4/object/1" {
+			t.Fatalf("Expected '1/2/3/4/object/1', got %s", nerr.Object)
+		}
+	}
+
+	_, err = obj.PutObject(bucketName, objectName+"/1/", 0, bytes.NewReader([]byte("")), nil, sha256sum)
+	if err == nil {
+		t.Fatal("Unexpected should fail here, backned corruption occurred")
+	}
+	if nerr, ok := errorCause(err).(PrefixAccessDenied); !ok {
+		t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
+	} else {
+		if nerr.Bucket != "bucket" {
+			t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
+		}
+		if nerr.Object != "1/2/3/4/object/1/" {
+			t.Fatalf("Expected '1/2/3/4/object/1/', got %s", nerr.Object)
+		}
+	}
+}
+
 // TestFSDeleteObject - test fs.DeleteObject() with healthy and corrupted disks
 func TestFSDeleteObject(t *testing.T) {
 	// Prepare for tests
@@ -172,7 +199,7 @@ func TestFSDeleteObject(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	obj.MakeBucket(bucketName)
+	obj.MakeBucketWithLocation(bucketName, "")
 	sha256sum := ""
 	obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
 
@@ -217,7 +244,7 @@ func TestFSDeleteBucket(t *testing.T) {
 	fs := obj.(*fsObjects)
 	bucketName := "bucket"
 
-	err := obj.MakeBucket(bucketName)
+	err := obj.MakeBucketWithLocation(bucketName, "")
 	if err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
@@ -235,7 +262,7 @@ func TestFSDeleteBucket(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	obj.MakeBucket(bucketName)
+	obj.MakeBucketWithLocation(bucketName, "")
 
 	// Delete bucker should get error disk not found.
 	removeAll(disk)
@@ -256,7 +283,7 @@ func TestFSListBuckets(t *testing.T) {
 	fs := obj.(*fsObjects)
 
 	bucketName := "bucket"
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -303,7 +330,7 @@ func TestFSHealObject(t *testing.T) {
 	defer removeAll(disk)
 
 	obj := initFSObjects(disk, t)
-	err := obj.HealObject("bucket", "object")
+	_, _, err := obj.HealObject("bucket", "object")
 	if err == nil || !isSameType(errorCause(err), NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/gateway-anonymous.go

@@ -0,0 +1,49 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "net/http"
+
+func anonErrToObjectErr(statusCode int, params ...string) error {
+	bucket := ""
+	object := ""
+	if len(params) >= 1 {
+		bucket = params[0]
+	}
+	if len(params) == 2 {
+		object = params[1]
+	}
+
+	switch statusCode {
+	case http.StatusNotFound:
+		if object != "" {
+			return ObjectNotFound{bucket, object}
+		}
+		return BucketNotFound{Bucket: bucket}
+	case http.StatusBadRequest:
+		if object != "" {
+			return ObjectNameInvalid{bucket, object}
+		}
+		return BucketNameInvalid{Bucket: bucket}
+	case http.StatusForbidden:
+		fallthrough
+	case http.StatusUnauthorized:
+		return AllAccessDisabled{bucket, object}
+	}
+
+	return errUnexpected
+}

cmd/gateway-azure-anonymous.go

@@ -29,14 +29,56 @@ import (
 	"github.com/Azure/azure-sdk-for-go/storage"
 )
 
+// Make anonymous HTTP request to azure endpoint.
+func azureAnonRequest(verb, urlStr string, header http.Header) (*http.Response, error) {
+	req, err := http.NewRequest(verb, urlStr, nil)
+	if err != nil {
+		return nil, err
+	}
+	if header != nil {
+		req.Header = header
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// 4XX and 5XX are error HTTP codes.
+	if resp.StatusCode >= 400 && resp.StatusCode <= 511 {
+		defer resp.Body.Close()
+		respBody, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(respBody) == 0 {
+			// no error in response body, might happen in HEAD requests
+			return nil, storage.AzureStorageServiceError{
+				StatusCode: resp.StatusCode,
+				Code:       resp.Status,
+				Message:    "no response body was available for error status code",
+			}
+		}
+		// Response contains Azure storage service error object.
+		var storageErr storage.AzureStorageServiceError
+		if err := xml.Unmarshal(respBody, &storageErr); err != nil {
+			return nil, err
+		}
+		storageErr.StatusCode = resp.StatusCode
+		return nil, storageErr
+	}
+
+	return resp, nil
+}
+
 // AnonGetBucketInfo - Get bucket metadata from azure anonymously.
-func (a AzureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
+func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
 	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
 	if err != nil {
 		return bucketInfo, azureToObjectError(traceError(err))
 	}
 	url.RawQuery = "restype=container"
-	resp, err := http.Head(url.String())
+	resp, err := azureAnonRequest(httpHEAD, url.String(), nil)
 	if err != nil {
 		return bucketInfo, azureToObjectError(traceError(err), bucket)
 	}
@@ -57,22 +99,24 @@ func (a AzureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, e
 	return bucketInfo, nil
 }
 
+// AnonPutObject - SendPUT request without authentication.
+// This is needed when clients send PUT requests on objects that can be uploaded without auth.
+func (a *azureObjects) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
+	// azure doesn't support anonymous put
+	return ObjectInfo{}, traceError(NotImplemented{})
+}
+
 // AnonGetObject - SendGET request without authentication.
 // This is needed when clients send GET requests on objects that can be downloaded without auth.
-func (a AzureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
-	url := a.client.GetBlobURL(bucket, object)
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return azureToObjectError(traceError(err), bucket, object)
-	}
-
+func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
+	h := make(http.Header)
 	if length > 0 && startOffset > 0 {
-		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
+		h.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
 	} else if startOffset > 0 {
-		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
+		h.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := azureAnonRequest(httpGET, a.client.GetBlobURL(bucket, object), h)
 	if err != nil {
 		return azureToObjectError(traceError(err), bucket, object)
 	}
@@ -88,8 +132,8 @@ func (a AzureObjects) AnonGetObject(bucket, object string, startOffset int64, le
 
 // AnonGetObjectInfo - Send HEAD request without authentication and convert the
 // result to ObjectInfo.
-func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
-	resp, err := http.Head(a.client.GetBlobURL(bucket, object))
+func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
+	resp, err := azureAnonRequest(httpHEAD, a.client.GetBlobURL(bucket, object), nil)
 	if err != nil {
 		return objInfo, azureToObjectError(traceError(err), bucket, object)
 	}
@@ -120,7 +164,7 @@ func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectIn
 		objInfo.UserDefined["Content-Encoding"] = resp.Header.Get("Content-Encoding")
 	}
 	objInfo.UserDefined["Content-Type"] = resp.Header.Get("Content-Type")
-	objInfo.MD5Sum = resp.Header.Get("Etag")
+	objInfo.ETag = resp.Header.Get("Etag")
 	objInfo.ModTime = t
 	objInfo.Name = object
 	objInfo.Size = contentLength
@@ -128,7 +172,7 @@ func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectIn
 }
 
 // AnonListObjects - Use Azure equivalent ListBlobs.
-func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
+func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
 	params := storage.ListBlobsParameters{
 		Prefix:     prefix,
 		Marker:     marker,
@@ -146,6 +190,63 @@ func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 	}
 	url.RawQuery = q.Encode()
 
+	resp, err := azureAnonRequest(httpGET, url.String(), nil)
+	if err != nil {
+		return result, azureToObjectError(traceError(err))
+	}
+	defer resp.Body.Close()
+
+	var listResp storage.BlobListResponse
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return result, azureToObjectError(traceError(err))
+	}
+	err = xml.Unmarshal(data, &listResp)
+	if err != nil {
+		return result, azureToObjectError(traceError(err))
+	}
+
+	result.IsTruncated = listResp.NextMarker != ""
+	result.NextMarker = listResp.NextMarker
+	for _, object := range listResp.Blobs {
+		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
+		if e != nil {
+			continue
+		}
+		result.Objects = append(result.Objects, ObjectInfo{
+			Bucket:          bucket,
+			Name:            object.Name,
+			ModTime:         t,
+			Size:            object.Properties.ContentLength,
+			ETag:            object.Properties.Etag,
+			ContentType:     object.Properties.ContentType,
+			ContentEncoding: object.Properties.ContentEncoding,
+		})
+	}
+	result.Prefixes = listResp.BlobPrefixes
+	return result, nil
+}
+
+// AnonListObjectsV2 - List objects in V2 mode, anonymously
+func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) {
+	params := storage.ListBlobsParameters{
+		Prefix:     prefix,
+		Marker:     continuationToken,
+		Delimiter:  delimiter,
+		MaxResults: uint(maxKeys),
+	}
+
+	q := azureListBlobsGetParameters(params)
+	q.Set("restype", "container")
+	q.Set("comp", "list")
+
+	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
+	if err != nil {
+		return result, azureToObjectError(traceError(err))
+	}
+	url.RawQuery = q.Encode()
+
 	resp, err := http.Get(url.String())
 	if err != nil {
 		return result, azureToObjectError(traceError(err))
@@ -163,8 +264,11 @@ func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 		return result, azureToObjectError(traceError(err))
 	}
 
-	result.IsTruncated = listResp.NextMarker != ""
-	result.NextMarker = listResp.NextMarker
+	// If NextMarker is not empty, this means response is truncated and NextContinuationToken should be set
+	if listResp.NextMarker != "" {
+		result.IsTruncated = true
+		result.NextContinuationToken = listResp.NextMarker
+	}
 	for _, object := range listResp.Blobs {
 		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
 		if e != nil {
@@ -175,7 +279,7 @@ func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 			Name:            object.Name,
 			ModTime:         t,
 			Size:            object.Properties.ContentLength,
-			MD5Sum:          object.Properties.Etag,
+			ETag:            canonicalizeETag(object.Properties.Etag),
 			ContentType:     object.Properties.ContentType,
 			ContentEncoding: object.Properties.ContentEncoding,
 		})

cmd/gateway-azure-unsupported.go

@@ -0,0 +1,43 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// HealBucket - Not relevant.
+func (a *azureObjects) HealBucket(bucket string) error {
+	return traceError(NotImplemented{})
+}
+
+// ListBucketsHeal - Not relevant.
+func (a *azureObjects) ListBucketsHeal() (buckets []BucketInfo, err error) {
+	return nil, traceError(NotImplemented{})
+}
+
+// HealObject - Not relevant.
+func (a *azureObjects) HealObject(bucket, object string) (int, int, error) {
+	return 0, 0, traceError(NotImplemented{})
+}
+
+// ListObjectsHeal - Not relevant.
+func (a *azureObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
+	return loi, traceError(NotImplemented{})
+}
+
+// ListUploadsHeal - Not relevant.
+func (a *azureObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
+	delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+	return lmi, traceError(NotImplemented{})
+}