Merge branch 'master' into release

This PR fixes the issue of cleaning up in-memory state
properly. Without this PR we can lead to security
situations where new bucket would inherit wrong
permissions on bucket and expose objects erroneously.

Fixes #4714

.travis.yml

@@ -18,6 +18,8 @@ env:
 script:
 ## Run all the tests
 - make
+- diff -au <(gofmt -d cmd) <(printf "")
+- diff -au <(gofmt -d pkg) <(printf "")
 - make test GOFLAGS="-timeout 15m -race -v"
 - make coverage
 

Dockerfile

@@ -1,5 +1,7 @@
 FROM alpine:3.5
 
+MAINTAINER Minio Inc <dev@minio.io>
+
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
@@ -9,10 +11,10 @@ WORKDIR /go/src/github.com/minio/
 RUN  \
      apk add --no-cache ca-certificates && \
      apk add --no-cache --virtual .build-deps git go musl-dev && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
-     git checkout release && \
-     go install -v -ldflags "-X github.com/minio/minio/cmd.Version=2017-05-05T01:14:51Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2017-05-05T01-14-51Z -X github.com/minio/minio/cmd.CommitID=40985cc4e3eec06b7ea82dc34c8d907fd2e7aa12" && \
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
      rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
 
 EXPOSE 9000

Dockerfile.aarch64

@@ -1,5 +1,7 @@
 FROM resin/aarch64-alpine:3.5
 
+MAINTAINER Minio Inc <dev@minio.io>
+
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
@@ -9,9 +11,10 @@ WORKDIR /go/src/github.com/minio/
 RUN  \
      apk add --no-cache ca-certificates && \
      apk add --no-cache --virtual .build-deps git go musl-dev && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \     
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "-X github.com/minio/minio/cmd.Version=2017-03-16T21:50:32Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2017-03-16T21-50-32Z -X github.com/minio/minio/cmd.CommitID=5311eb22fd681a8cd4a46e2a872d46c2352c64e8" && \
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
      rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
 
 EXPOSE 9000

Dockerfile.armhf

@@ -1,5 +1,7 @@
 FROM resin/armhf-alpine:3.5
 
+MAINTAINER Minio Inc <dev@minio.io>
+
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
@@ -9,9 +11,10 @@ WORKDIR /go/src/github.com/minio/
 RUN  \
      apk add --no-cache ca-certificates && \
      apk add --no-cache --virtual .build-deps git go musl-dev && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "-X github.com/minio/minio/cmd.Version=2017-03-16T21:50:32Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2017-03-16T21-50-32Z -X github.com/minio/minio/cmd.CommitID=5311eb22fd681a8cd4a46e2a872d46c2352c64e8" && \
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
      rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
 
 EXPOSE 9000

Dockerfile.release

@@ -0,0 +1,26 @@
+FROM alpine:3.5
+
+MAINTAINER Minio Inc <dev@minio.io>
+ 
+COPY buildscripts/docker-entrypoint.sh buildscripts/healthcheck.sh /usr/bin/ 
+
+RUN \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps curl && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
+CMD ["minio"]
+

Dockerfile.release.aarch64

@@ -0,0 +1,25 @@
+FROM resin/aarch64-alpine:3.5
+
+MAINTAINER Minio Inc <dev@minio.io>
+
+COPY buildscripts/docker-entrypoint.sh buildscripts/healthcheck.sh /usr/bin/ 
+
+RUN \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps curl && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.minio.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
+CMD ["minio"]

Dockerfile.release.armhf

@@ -0,0 +1,25 @@
+FROM resin/armhf-alpine:3.5
+
+MAINTAINER Minio Inc <dev@minio.io>
+
+COPY buildscripts/docker-entrypoint.sh buildscripts/healthcheck.sh /usr/bin/ 
+
+RUN \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps curl && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.minio.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
+CMD ["minio"]

Makefile

@@ -1,112 +1,49 @@
-LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 PWD := $(shell pwd)
 GOPATH := $(shell go env GOPATH)
-BUILD_LDFLAGS := '$(LDFLAGS) -s -w'
-TAG := latest
+LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 
-HOST ?= $(shell uname)
-CPU ?= $(shell uname -m)
+BUILD_LDFLAGS := '$(LDFLAGS)'
 
-# if no host is identifed (no uname tool)
-# we assume a Linux-64bit build
-ifeq ($(HOST),)
-  HOST = Linux
-endif
-
-# identify CPU
-ifeq ($(CPU), x86_64)
-  HOST := $(HOST)64
-else
-ifeq ($(CPU), amd64)
-  HOST := $(HOST)64
-else
-ifeq ($(CPU), i686)
-  HOST := $(HOST)32
-endif
-endif
-endif
-
-
-#############################################
-# now we find out the target OS for
-# which we are going to compile in case
-# the caller didn't yet define OS himself
-ifndef (OS)
-  ifeq ($(HOST), Linux64)
-    arch = gcc
-  else
-  ifeq ($(HOST), Linux32)
-    arch = 32
-  else
-  ifeq ($(HOST), Darwin64)
-    arch = clang
-  else
-  ifeq ($(HOST), Darwin32)
-    arch = clang
-  else
-  ifeq ($(HOST), FreeBSD64)
-    arch = gcc
-  endif
-  endif
-  endif
-  endif
-  endif
-endif
-
-all: install
+all: build
 
 checks:
-	@echo -n "Check deps: "
+	@echo "Check deps"
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
-	@echo "Done."
-	@echo -n "Checking project is in GOPATH: "
+	@echo "Checking project is in GOPATH"
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
-	@echo "Done."
 
 getdeps: checks
-	@echo -n "Installing golint: " && go get -u github.com/golang/lint/golint
-	@echo "Done."
-	@echo -n "Installing gocyclo: " && go get -u github.com/fzipp/gocyclo
-	@echo "Done."
-	@echo -n "Installing deadcode: " && go get -u github.com/remyoudompheng/go-misc/deadcode
-	@echo "Done."
-	@echo -n "Installing misspell: " && go get -u github.com/client9/misspell/cmd/misspell
-	@echo "Done."
-	@echo -n "Installing ineffassign: " && go get -u github.com/gordonklaus/ineffassign
-	@echo "Done."
+	@echo "Installing golint" && go get -u github.com/golang/lint/golint
+	@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
+	@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
+	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
+	@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
 
-verifiers: vet fmt lint cyclo spelling
+verifiers: getdeps vet fmt lint cyclo spelling
 
 vet:
-	@echo -n "Running $@: "
+	@echo "Running $@"
 	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult cmd
 	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult pkg
-	@echo "Done."
 
 fmt:
-	@echo -n "Running $@: "
-	@gofmt -s -l cmd
-	@gofmt -s -l pkg
-	@echo "Done."
+	@echo "Running $@"
+	@gofmt -d cmd
+	@gofmt -d pkg
 
 lint:
-	@echo -n "Running $@: "
+	@echo "Running $@"
 	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd...
 	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg...
-	@echo "Done."
 
 ineffassign:
-	@echo -n "Running $@: "
+	@echo "Running $@"
 	@${GOPATH}/bin/ineffassign .
-	@echo "Done."
 
 cyclo:
-	@echo -n "Running $@: "
+	@echo "Running $@"
 	@${GOPATH}/bin/gocyclo -over 100 cmd
 	@${GOPATH}/bin/gocyclo -over 100 pkg
-	@echo "Done."
-
-build: getdeps verifiers $(UI_ASSETS)
 
 deadcode:
 	@${GOPATH}/bin/deadcode
@@ -116,36 +53,43 @@ spelling:
 	@${GOPATH}/bin/misspell -error `find pkg/`
 	@${GOPATH}/bin/misspell -error `find docs/`
 
-test: build
-	@echo -n "Running all minio testing: "
+# Builds minio, runs the verifiers then runs the tests.
+check: test
+test: verifiers build
+	@echo "Running all minio testing"
 	@go test $(GOFLAGS) .
 	@go test $(GOFLAGS) github.com/minio/minio/cmd...
 	@go test $(GOFLAGS) github.com/minio/minio/pkg...
-	@echo "Done."
 
 coverage: build
-	@echo -n "Running all coverage for minio: "
+	@echo "Running all coverage for minio"
 	@./buildscripts/go-coverage.sh
-	@echo "Done."
 
-gomake-all: build
-	@echo -n "Installing minio at $(GOPATH)/bin/minio: "
-	@go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
-	@echo "Done."
+# Builds minio locally.
+build:
+	@echo "Building minio to $(PWD)/minio ..."
+	@CGO_ENABLED=0 go build --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
 
 pkg-add:
-	${GOPATH}/bin/govendor add $(PKG)
+	@echo "Adding new package $(PKG)"
+	@${GOPATH}/bin/govendor add $(PKG)
 
 pkg-update:
-	${GOPATH}/bin/govendor update $(PKG)
+	@echo "Updating new package $(PKG)"
+	@${GOPATH}/bin/govendor update $(PKG)
 
 pkg-remove:
-	${GOPATH}/bin/govendor remove $(PKG)
+	@echo "Remove new package $(PKG)"
+	@${GOPATH}/bin/govendor remove $(PKG)
 
 pkg-list:
 	@$(GOPATH)/bin/govendor list
 
-install: gomake-all
+# Builds minio and installs it to $GOPATH/bin.
+install: build
+	@echo "Installing minio at $(GOPATH)/bin/minio ..."
+	@cp $(PWD)/minio $(GOPATH)/bin/minio
+	@echo "Check 'minio -h' for help."
 
 release: verifiers
 	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
@@ -154,8 +98,7 @@ experimental: verifiers
 	@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
 
 clean:
-	@echo -n "Cleaning up all the generated files: "
+	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
 	@rm -rf build
 	@rm -rf release
-	@echo "Done."

README.md

@@ -26,7 +26,13 @@ Install minio packages using [Homebrew](http://brew.sh/)
 brew install minio/stable/minio
 minio server ~/Photos
 ```
-Note: If you are upgrading minio on macOS, please see instructions [here](https://github.com/minio/minio/blob/master/docs/minio_homebrew.md).
+#### Note
+If you previously installed minio using `brew install minio` then reinstall minio from `minio/stable/minio` official repo. Homebrew builds are unstable due to golang 1.8 bugs.
+
+```
+brew uninstall minio 
+brew install minio/stable/minio
+```  
 
 ### Binary Download
 | Platform| Architecture | URL|

appveyor.yml

@@ -24,6 +24,8 @@ install:
 # To run your custom scripts instead of automatic MSBuild
 build_script:
   # Compile
+  # We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
+  - ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
   - appveyor AddCompilationMessage "Starting Compile"
   - cd c:\gopath\src\github.com\minio\minio
   - go run buildscripts/gen-ldflags.go > temp.txt
@@ -36,9 +38,9 @@ test_script:
   # Unit tests
   - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
   - mkdir build\coverage
-  - go test -timeout 17m -race github.com/minio/minio/cmd...
-  - go test -race github.com/minio/minio/pkg...
-  - go test -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
+  - go test -v -timeout 17m -race github.com/minio/minio/cmd...
+  - go test -v -race github.com/minio/minio/pkg...
+  - go test -v -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
   - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
 
 after_test:

browser/README.md

@@ -27,7 +27,7 @@ go get github.com/elazarl/go-bindata-assetfs/...
 yarn release
 ```
 
-This generates ui-assets.go in the current direcotry. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
+This generates ui-assets.go in the current directory. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
 
 ### Run Minio Browser with live reload
 

browser/app/js/actions.js

@@ -432,6 +432,8 @@ export const setLoginError = () => {
 
 export const downloadSelected = (url, req, xhr) => {
   return (dispatch) => {
+    var anchor = document.createElement('a')
+    document.body.appendChild(anchor);
     xhr.open('POST', url, true)
     xhr.responseType = 'blob'
 
@@ -439,10 +441,20 @@ export const downloadSelected = (url, req, xhr) => {
       if (this.status == 200) {
         dispatch(checkedObjectsReset())
         var blob = new Blob([this.response], {
-          type: 'application/zip'
+          type: 'octet/stream'
         })
         var blobUrl = window.URL.createObjectURL(blob);
-        window.location = blobUrl
+        var separator = req.prefix.length > 1 ? '-' : ''
+
+        anchor.href = blobUrl
+        anchor.download = req.bucketName+separator+req.prefix.slice(0, -1)+'.zip';
+
+
+
+
+        anchor.click()
+        window.URL.revokeObjectURL(blobUrl)
+        anchor.remove()
       }
     };
     xhr.send(JSON.stringify(req));

browser/app/js/components/Browse.js

@@ -68,7 +68,7 @@ export default class Browse extends React.Component {
           memory: res.MinioMemory,
           platform: res.MinioPlatform,
           runtime: res.MinioRuntime,
-          envVars: res.MinioEnvVars
+          info: res.MinioGlobalInfo
         })
         dispatch(actions.setServerInfo(serverInfo))
       })
@@ -370,8 +370,24 @@ export default class Browse extends React.Component {
   }
 
   handleExpireValue(targetInput, inc, object) {
-    inc === -1 ? this.refs[targetInput].stepDown(1) : this.refs[targetInput].stepUp(1)
+    let value = this.refs[targetInput].value
+    let maxValue = (targetInput == 'expireHours') ? 23 : (targetInput == 'expireMins') ? 59 : (targetInput == 'expireDays') ? 7 : 0
+    value = isNaN(value) ? 0 : value
 
+    // Use custom step count to support browser Edge
+    if((inc === -1)) {
+      if(value != 0) {
+        value--
+      }
+    }
+    else {
+      if(value != maxValue) {
+        value++
+      }
+    }
+    this.refs[targetInput].value = value
+
+    // Reset hours and mins when days reaches it's max value
     if (this.refs.expireDays.value == 7) {
       this.refs.expireHours.value = 0
       this.refs.expireMins.value = 0
@@ -379,6 +395,7 @@ export default class Browse extends React.Component {
     if (this.refs.expireDays.value + this.refs.expireHours.value + this.refs.expireMins.value == 0) {
       this.refs.expireDays.value = 7
     }
+
     const {dispatch} = this.props
     dispatch(actions.shareObject(object, this.refs.expireDays.value, this.refs.expireHours.value, this.refs.expireMins.value))
   }
@@ -463,21 +480,25 @@ export default class Browse extends React.Component {
     }
 
     if (web.LoggedIn()) {
-      storageUsageDetails = <div className="feh-usage">
-                              <div className="fehu-chart">
-                                <div style={ { width: usedPercent } }></div>
-                              </div>
-                              <ul>
-                                <li>
-                                  <span>Used: </span>
-                                  { humanize.filesize(total - free) }
-                                </li>
-                                <li className="pull-right">
-                                  <span>Free: </span>
-                                  { humanize.filesize(total - used) }
-                                </li>
-                              </ul>
-                            </div>
+      if (!(used === 0 && free === 0)) {
+        storageUsageDetails = <div className="feh-usage">
+          <div className="fehu-chart">
+            <div style={ { width: usedPercent } }></div>
+          </div>
+          <ul>
+            <li>
+              <span>Used: </span>
+              { humanize.filesize(total - free) }
+            </li>
+            <li className="pull-right">
+              <span>Free: </span>
+              { humanize.filesize(total - used) }
+            </li>
+          </ul>
+        </div>
+      }
+
+
     }
 
     let createButton = ''
@@ -722,11 +743,11 @@ export default class Browse extends React.Component {
                 </div>
                 <div className="input-group" style={ { display: web.LoggedIn() ? 'block' : 'none' } }>
                   <label>
-                    Expires in
+                    Expires in (Max 7 days)
                   </label>
                   <div className="set-expire">
                     <div className="set-expire-item">
-                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireDays', 1, shareObject.object) }></i>
+                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireDays', 1, shareObject.object) } />
                       <div className="set-expire-title">
                         Days
                       </div>
@@ -735,12 +756,14 @@ export default class Browse extends React.Component {
                           type="number"
                           min={ 0 }
                           max={ 7 }
-                          defaultValue={ 5 } />
+                          defaultValue={ 5 }
+                          readOnly="readOnly"
+                        />
                       </div>
-                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireDays', -1, shareObject.object) }></i>
+                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireDays', -1, shareObject.object) } />
                     </div>
                     <div className="set-expire-item">
-                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireHours', 1, shareObject.object) }></i>
+                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireHours', 1, shareObject.object) } />
                       <div className="set-expire-title">
                         Hours
                       </div>
@@ -749,12 +772,14 @@ export default class Browse extends React.Component {
                           type="number"
                           min={ 0 }
                           max={ 23 }
-                          defaultValue={ 0 } />
+                          defaultValue={ 0 }
+                          readOnly="readOnly"
+                        />
                       </div>
-                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireHours', -1, shareObject.object) }></i>
+                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireHours', -1, shareObject.object) } />
                     </div>
                     <div className="set-expire-item">
-                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireMins', 1, shareObject.object) }></i>
+                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireMins', 1, shareObject.object) } />
                       <div className="set-expire-title">
                         Minutes
                       </div>
@@ -763,9 +788,11 @@ export default class Browse extends React.Component {
                           type="number"
                           min={ 0 }
                           max={ 59 }
-                          defaultValue={ 0 } />
+                          defaultValue={ 0 }
+                          readOnly="readOnly"
+                        />
                       </div>
-                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireMins', -1, shareObject.object) }></i>
+                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireMins', -1, shareObject.object) } />
                     </div>
                   </div>
                 </div>

browser/app/js/components/PolicyInput.js

@@ -7,6 +7,7 @@ import * as actions from '../actions'
 class PolicyInput extends Component {
   componentDidMount() {
     const {web, dispatch} = this.props
+    this.prefix.focus()
     web.ListAllBucketPolicies({
       bucketName: this.props.currentBucket
     }).then(res => {
@@ -27,8 +28,23 @@ class PolicyInput extends Component {
 
   handlePolicySubmit(e) {
     e.preventDefault()
-    const {web, dispatch} = this.props
+    const {web, dispatch, currentBucket} = this.props
 
+    let prefix = currentBucket + '/' + this.prefix.value
+    let policy = this.policy.value
+
+    if (!prefix.endsWith('*')) prefix = prefix + '*'
+ 
+    let prefixAlreadyExists = this.props.policies.some(elem => prefix === elem.prefix)
+
+    if (prefixAlreadyExists) {
+      dispatch(actions.showAlert({
+       type: 'danger',
+       message: "Policy for this prefix already exists."
+      }))
+      return
+    }
+    
     web.SetBucketPolicy({
       bucketName: this.props.currentBucket,
       prefix: this.prefix.value,
@@ -36,8 +52,7 @@ class PolicyInput extends Component {
     })
       .then(() => {
         dispatch(actions.setPolicies([{
-          policy: this.policy.value,
-          prefix: this.prefix.value + '*',
+          policy, prefix
         }, ...this.props.policies]))
         this.prefix.value = ''
       })

browser/app/js/components/SettingsModal.js

@@ -34,23 +34,12 @@ class SettingsModal extends React.Component {
 
     let accessKeyEnv = ''
     let secretKeyEnv = ''
-    // Check environment variables first. They may or may not have been
-    // loaded already; they load in Browse#componentDidMount.
-    if (serverInfo.envVars) {
-      serverInfo.envVars.forEach(envVar => {
-        let keyVal = envVar.split('=')
-        if (keyVal[0] == 'MINIO_ACCESS_KEY') {
-          accessKeyEnv = keyVal[1]
-        } else if (keyVal[0] == 'MINIO_SECRET_KEY') {
-          secretKeyEnv = keyVal[1]
-        }
-      })
-    }
-    if (accessKeyEnv != '' || secretKeyEnv != '') {
+    // Check environment variables first.
+    if (serverInfo.info.isEnvCreds) {
       dispatch(actions.setSettings({
-        accessKey: accessKeyEnv,
-        secretKey: secretKeyEnv,
-        keysReadOnly: true
+          accessKey: 'xxxxxxxxx',
+          secretKey: 'xxxxxxxxx',
+          keysReadOnly: true
       }))
     } else {
       web.GetAuth()

browser/app/less/inc/form.less

@@ -183,6 +183,17 @@ select.form-control {
 .set-expire {
     border: 1px solid @input-border;
     margin: 35px 0 30px;
+    position: relative;
+
+    &:before {
+        content: '';
+        position: absolute;
+        left: 0;
+        top: 0;
+        width: 100%;
+        height: 100%;
+        z-index: 1;
+    }
 }
 
 .set-expire-item {
@@ -191,6 +202,7 @@ select.form-control {
     display: table-cell;
     width: 1%;
     text-align: center;
+    .user-select(none);
 
     &:not(:last-child) {
         border-right: 1px solid @input-border;
@@ -209,6 +221,7 @@ select.form-control {
     left: -8px;
 
     input {
+        .user-select(none);
         font-size: 20px;
         text-align: center;
         position: relative;

buildscripts/build.sh

@@ -5,7 +5,7 @@ _init() {
     LDFLAGS=$(go run buildscripts/gen-ldflags.go)
 
     # Extract release tag
-    release_tag=$(echo $LDFLAGS | awk {'print $4'} | cut -f2 -d=)
+    release_tag=$(echo $LDFLAGS | awk {'print $6'} | cut -f2 -d=)
 
     # Verify release tag.
     if [ -z "$release_tag" ]; then
@@ -28,6 +28,7 @@ _init() {
     ## System binaries
     CP=`which cp`
     SHASUM=`which shasum`
+    SHA256SUM="${SHASUM} -a 256"
     SED=`which sed`
 }
 
@@ -43,58 +44,36 @@ go_build() {
     release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
     # Release binary downloadable name
     release_real_bin="$release_str/$os-$arch/$(basename $package)"
-    # Release shasum name
-    release_shasum="$release_str/$os-$arch/$(basename $package).shasum"
+
+    # Release sha1sum name
+    release_shasum="$release_str/$os-$arch/$(basename $package).${release_tag}.shasum"
+    # Release sha1sum default
+    release_shasum_default="$release_str/$os-$arch/$(basename $package).shasum"
+
+    # Release sha256sum name
+    release_sha256sum="$release_str/$os-$arch/$(basename $package).${release_tag}.sha256sum"
+    # Release sha256sum default
+    release_sha256sum_default="$release_str/$os-$arch/$(basename $package).sha256sum"
 
     # Go build to build the binary.
-    if [ "${arch}" == "arm" ]; then
-        # Release binary downloadable name
-        release_real_bin_6="$release_str/$os-${arch}6vl/$(basename $package)"
+    CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
 
-        release_bin_6="$release_str/$os-${arch}6vl/$(basename $package).$release_tag"
-        ## Support building for ARM6vl
-        GOARM=6 GOOS=$os GOARCH=$arch go build --ldflags "-s -w ${LDFLAGS}" -o $release_bin_6
-
-        ## Copy
-        $CP -p $release_bin_6 $release_real_bin_6
-
-        # Release shasum name
-        release_shasum_6="$release_str/$os-${arch}6vl/$(basename $package).shasum"
-
-        # Calculate shasum
-        shasum_str=$(${SHASUM} ${release_bin_6})
-        echo ${shasum_str} | $SED "s/$release_str\/$os-${arch}6vl\///g" > $release_shasum_6
-
-        # Release binary downloadable name
-        release_real_bin_7="$release_str/$os-$arch/$(basename $package)"
-
-        release_bin_7="$release_str/$os-$arch/$(basename $package).$release_tag"
-        ## Support building for ARM7vl
-        GOARM=7 GOOS=$os GOARCH=$arch go build --ldflags "-s -w ${LDFLAGS}" -o $release_bin_7
-
-        ## Copy
-        $CP -p $release_bin_7 $release_real_bin_7
-
-        # Release shasum name
-        release_shasum_7="$release_str/$os-$arch/$(basename $package).shasum"
-
-        # Calculate shasum
-        shasum_str=$(${SHASUM} ${release_bin_7})
-        echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum_7
+    # Create copy
+    if [ $os == "windows" ]; then
+        $CP -p $release_bin ${release_real_bin}.exe
     else
-        GOOS=$os GOARCH=$arch go build --ldflags "-s -w ${LDFLAGS}" -o $release_bin
-
-        # Create copy
-        if [ $os == "windows" ]; then
-            $CP -p $release_bin ${release_real_bin}.exe
-        else
-            $CP -p $release_bin $release_real_bin
-        fi
-
-        # Calculate shasum
-        shasum_str=$(${SHASUM} ${release_bin})
-        echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
+        $CP -p $release_bin $release_real_bin
     fi
+
+    # Calculate sha1sum
+    shasum_str=$(${SHASUM} ${release_bin})
+    echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
+    $CP -p $release_shasum $release_shasum_default
+
+    # Calculate sha256sum
+    sha256sum_str=$(${SHA256SUM} ${release_bin})
+    echo ${sha256sum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_sha256sum
+    $CP -p $release_sha256sum $release_sha256sum_default
 }
 
 main() {
@@ -111,9 +90,18 @@ main() {
             go_build ${each_osarch}
         done
     else
-        for each_osarch in $(echo $chosen_osarch | sed 's/,/ /g'); do
-            go_build ${each_osarch}
+        local found=0
+        for each_osarch in ${SUPPORTED_OSARCH}; do
+            if [ "$chosen_osarch" = "$each_osarch" ]; then
+                found=1
+            fi
         done
+        if [ ${found} -eq 1 ]; then
+            go_build ${chosen_osarch}
+        else
+            echo "Unknown architecture \"${chosen_osarch}\""
+            exit 1
+        fi
     fi
 
 }

buildscripts/healthcheck.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# Minio Cloud Storage, (C) 2017 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+_init () {
+    address="http://127.0.0.1:9000"
+    resource="/minio/index.html"
+}
+
+HealthCheckMain () {
+    # Get the http response code
+    http_response=$(curl -s -o /dev/null -I -w "%{http_code}" ${address}${resource})
+
+    # If http_repsonse is 200 - server is up. 
+    # When MINIO_BROWSER is set to off, curl responds with 404. We assume that the the server is up
+    [ "$http_response" == "200" ] || [ "$http_response" == "404" ]
+}
+
+_init && HealthCheckMain

cmd/admin-handlers.go

@@ -241,9 +241,9 @@ type ServerInfoData struct {
 
 // ServerInfo holds server information result of one node
 type ServerInfo struct {
-	Error error
-	Addr  string
-	Data  *ServerInfoData
+	Error string          `json:"error"`
+	Addr  string          `json:"addr"`
+	Data  *ServerInfoData `json:"data"`
 }
 
 // ServerInfoHandler - GET /?info
@@ -276,7 +276,7 @@ func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *htt
 			serverInfoData, err := peer.cmdRunner.ServerInfoData()
 			if err != nil {
 				errorIf(err, "Unable to get server info from %s.", peer.addr)
-				reply[idx].Error = err
+				reply[idx].Error = err.Error()
 				return
 			}
 

cmd/admin-handlers_test.go

@@ -731,7 +731,7 @@ func TestListObjectsHealHandler(t *testing.T) {
 	}
 	defer adminTestBed.TearDown()
 
-	err = adminTestBed.objLayer.MakeBucket("mybucket")
+	err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}
@@ -859,7 +859,7 @@ func TestHealBucketHandler(t *testing.T) {
 	}
 	defer adminTestBed.TearDown()
 
-	err = adminTestBed.objLayer.MakeBucket("mybucket")
+	err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}
@@ -936,7 +936,7 @@ func TestHealObjectHandler(t *testing.T) {
 	// Create an object myobject under bucket mybucket.
 	bucketName := "mybucket"
 	objName := "myobject"
-	err = adminTestBed.objLayer.MakeBucket(bucketName)
+	err = adminTestBed.objLayer.MakeBucketWithLocation(bucketName, "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
 	}
@@ -1067,7 +1067,7 @@ func TestHealUploadHandler(t *testing.T) {
 	// Create an object myobject under bucket mybucket.
 	bucketName := "mybucket"
 	objName := "myobject"
-	err = adminTestBed.objLayer.MakeBucket(bucketName)
+	err = adminTestBed.objLayer.MakeBucketWithLocation(bucketName, "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
 	}
@@ -1314,7 +1314,7 @@ func TestAdminServerInfo(t *testing.T) {
 		if len(serverInfo.Addr) == 0 {
 			t.Error("Expected server address to be non empty")
 		}
-		if serverInfo.Error != nil {
+		if serverInfo.Error != "" {
 			t.Errorf("Unexpected error = %v\n", serverInfo.Error)
 		}
 		if serverInfo.Data.StorageInfo.Free == 0 {
@@ -1455,7 +1455,7 @@ func TestListHealUploadsHandler(t *testing.T) {
 	}
 	defer adminTestBed.TearDown()
 
-	err = adminTestBed.objLayer.MakeBucket("mybucket")
+	err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}

cmd/admin-rpc-client.go

@@ -113,15 +113,15 @@ func (rc remoteAdminClient) ReInitDisks() error {
 }
 
 // ServerInfoData - Returns the server info of this server.
-func (lc localAdminClient) ServerInfoData() (ServerInfoData, error) {
+func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
 	if globalBootTime.IsZero() {
-		return ServerInfoData{}, errServerNotInitialized
+		return sid, errServerNotInitialized
 	}
 
 	// Build storage info
 	objLayer := newObjectLayerFn()
 	if objLayer == nil {
-		return ServerInfoData{}, errServerNotInitialized
+		return sid, errServerNotInitialized
 	}
 	storage := objLayer.StorageInfo()
 
@@ -145,12 +145,12 @@ func (lc localAdminClient) ServerInfoData() (ServerInfoData, error) {
 }
 
 // ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rc remoteAdminClient) ServerInfoData() (ServerInfoData, error) {
+func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
 	args := AuthRPCArgs{}
 	reply := ServerInfoDataReply{}
 	err := rc.Call(serverInfoDataRPC, &args, &reply)
 	if err != nil {
-		return ServerInfoData{}, err
+		return sid, err
 	}
 
 	return reply.ServerInfoData, nil
@@ -493,7 +493,7 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
 
 // getValidServerConfig - finds the server config that is present in
 // quorum or more number of servers.
-func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (serverConfigV13, error) {
+func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (scv serverConfigV13, e error) {
 	// majority-based quorum
 	quorum := len(serverConfigs)/2 + 1
 
@@ -566,7 +566,7 @@ func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (server
 
 	// If quorum nodes don't agree.
 	if maxOccurrence < quorum {
-		return serverConfigV13{}, errXLWriteQuorum
+		return scv, errXLWriteQuorum
 	}
 
 	return configJSON, nil

cmd/admin-rpc-server.go

@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"net/rpc"
 	"os"
 	"path/filepath"
 	"time"
@@ -236,7 +235,7 @@ func (s *adminCmd) CommitConfig(cArgs *CommitConfigArgs, cReply *CommitConfigRep
 // stop and restart commands.
 func registerAdminRPCRouter(mux *router.Router) error {
 	adminRPCHandler := &adminCmd{}
-	adminRPCServer := rpc.NewServer()
+	adminRPCServer := newRPCServer()
 	err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
 	if err != nil {
 		return traceError(err)

cmd/api-errors.go

@@ -115,6 +115,7 @@ const (
 	ErrBucketAlreadyOwnedByYou
 	ErrInvalidDuration
 	ErrNotSupported
+	ErrBucketAlreadyExists
 	// Add new error codes here.
 
 	// Bucket notification related errors.
@@ -149,6 +150,7 @@ const (
 	ErrAdminInvalidAccessKey
 	ErrAdminInvalidSecretKey
 	ErrAdminConfigNoQuorum
+	ErrInsecureClientRequest
 )
 
 // error code to APIError structure, these fields carry respective
@@ -230,7 +232,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
 	ErrInvalidAccessKeyID: {
-		Code:           "InvalidAccessKeyID",
+		Code:           "InvalidAccessKeyId",
 		Description:    "The access key ID you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
@@ -316,7 +318,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInvalidPart: {
 		Code:           "InvalidPart",
-		Description:    "One or more of the specified parts could not be found.",
+		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartOrder: {
@@ -354,6 +356,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The bucket you tried to delete is not empty",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrBucketAlreadyExists: {
+		Code:           "BucketAlreadyExists",
+		Description:    "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again.",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrAllAccessDisabled: {
 		Code:           "AllAccessDisabled",
 		Description:    "All access to this bucket has been disabled.",
@@ -618,6 +625,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Configuration update failed because server quorum was not met",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrInsecureClientRequest: {
+		Code:           "XMinioInsecureClientRequest",
+		Description:    "Cannot respond to plain-text request from TLS-encrypted server",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// Add your error structure here.
 }
@@ -657,6 +669,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrStorageFull
 	case BadDigest:
 		apiErr = ErrBadDigest
+	case AllAccessDisabled:
+		apiErr = ErrAllAccessDisabled
 	case IncompleteBody:
 		apiErr = ErrIncompleteBody
 	case ObjectExistsAsDirectory:
@@ -671,6 +685,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrBucketAlreadyOwnedByYou
 	case BucketNotEmpty:
 		apiErr = ErrBucketNotEmpty
+	case BucketAlreadyExists:
+		apiErr = ErrBucketAlreadyExists
 	case BucketExists:
 		apiErr = ErrBucketAlreadyOwnedByYou
 	case ObjectNotFound:
@@ -695,6 +711,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrNoSuchUpload
 	case PartTooSmall:
 		apiErr = ErrEntityTooSmall
+	case SignatureDoesNotMatch:
+		apiErr = ErrSignatureDoesNotMatch
 	case SHA256Mismatch:
 		apiErr = ErrContentSHA256Mismatch
 	case ObjectTooLarge:
@@ -707,6 +725,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrNotImplemented
 	case PolicyNotFound:
 		apiErr = ErrNoSuchBucketPolicy
+	case PartTooBig:
+		apiErr = ErrEntityTooLarge
 	default:
 		apiErr = ErrInternalError
 	}

cmd/api-headers.go

@@ -36,7 +36,11 @@ func setCommonHeaders(w http.ResponseWriter) {
 	// Set unique request ID for each reply.
 	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
 	w.Header().Set("Server", globalServerUserAgent)
-	w.Header().Set("X-Amz-Bucket-Region", serverConfig.GetRegion())
+	// Set `x-amz-bucket-region` only if region is set on the server
+	// by default minio uses an empty region.
+	if region := serverConfig.GetRegion(); region != "" {
+		w.Header().Set("X-Amz-Bucket-Region", region)
+	}
 	w.Header().Set("Accept-Ranges", "bytes")
 }
 
@@ -62,8 +66,8 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 	w.Header().Set("Last-Modified", lastModified)
 
 	// Set Etag if available.
-	if objInfo.MD5Sum != "" {
-		w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+	if objInfo.ETag != "" {
+		w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
 	}
 
 	// Set all other user defined metadata.

cmd/api-response.go

@@ -288,8 +288,6 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	var owner = Owner{}
 
 	owner.ID = globalMinioDefaultOwnerID
-	owner.DisplayName = globalMinioDefaultOwnerID
-
 	for _, bucket := range buckets {
 		var listbucket = Bucket{}
 		listbucket.Name = bucket.Name
@@ -312,8 +310,6 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 	var data = ListObjectsResponse{}
 
 	owner.ID = globalMinioDefaultOwnerID
-	owner.DisplayName = globalMinioDefaultOwnerID
-
 	for _, object := range resp.Objects {
 		var content = Object{}
 		if object.Name == "" {
@@ -321,8 +317,8 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 		}
 		content.Key = object.Name
 		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
-		if object.MD5Sum != "" {
-			content.ETag = "\"" + object.MD5Sum + "\""
+		if object.ETag != "" {
+			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
 		content.StorageClass = globalMinioDefaultStorageClass
@@ -352,26 +348,25 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, fetchOwner bool, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
 	var contents []Object
-	var prefixes []CommonPrefix
+	var commonPrefixes []CommonPrefix
 	var owner = Owner{}
 	var data = ListObjectsV2Response{}
 
 	if fetchOwner {
 		owner.ID = globalMinioDefaultOwnerID
-		owner.DisplayName = globalMinioDefaultOwnerID
 	}
 
-	for _, object := range resp.Objects {
+	for _, object := range objects {
 		var content = Object{}
 		if object.Name == "" {
 			continue
 		}
 		content.Key = object.Name
 		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
-		if object.MD5Sum != "" {
-			content.ETag = "\"" + object.MD5Sum + "\""
+		if object.ETag != "" {
+			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
 		content.StorageClass = globalMinioDefaultStorageClass
@@ -387,14 +382,14 @@ func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter
 	data.Prefix = prefix
 	data.MaxKeys = maxKeys
 	data.ContinuationToken = token
-	data.NextContinuationToken = resp.NextMarker
-	data.IsTruncated = resp.IsTruncated
-	for _, prefix := range resp.Prefixes {
+	data.NextContinuationToken = nextToken
+	data.IsTruncated = isTruncated
+	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = prefix
-		prefixes = append(prefixes, prefixItem)
+		commonPrefixes = append(commonPrefixes, prefixItem)
 	}
-	data.CommonPrefixes = prefixes
+	data.CommonPrefixes = commonPrefixes
 	data.KeyCount = len(data.Contents) + len(data.CommonPrefixes)
 	return data
 }
@@ -443,9 +438,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
 	listPartsResponse.UploadID = partsInfo.UploadID
 	listPartsResponse.StorageClass = globalMinioDefaultStorageClass
 	listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID
-	listPartsResponse.Initiator.DisplayName = globalMinioDefaultOwnerID
 	listPartsResponse.Owner.ID = globalMinioDefaultOwnerID
-	listPartsResponse.Owner.DisplayName = globalMinioDefaultOwnerID
 
 	listPartsResponse.MaxParts = partsInfo.MaxParts
 	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker

cmd/auth-handler.go

@@ -113,13 +113,13 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string)
 		// Signature V2 validation.
 		s3Error := isReqAuthenticatedV2(r)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 		}
 		return s3Error
 	case authTypeSigned, authTypePresigned:
 		s3Error := isReqAuthenticated(r, region)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 		}
 		return s3Error
 	}

cmd/auth-rpc-client.go

@@ -52,10 +52,10 @@ type authConfig struct {
 
 // AuthRPCClient is a authenticated RPC client which does authentication before doing Call().
 type AuthRPCClient struct {
-	sync.Mutex            // Mutex to lock this object.
-	rpcClient  *RPCClient // Reconnectable RPC client to make any RPC call.
-	config     authConfig // Authentication configuration information.
-	authToken  string     // Authentication token.
+	sync.RWMutex            // Mutex to lock this object.
+	rpcClient    *RPCClient // Reconnectable RPC client to make any RPC call.
+	config       authConfig // Authentication configuration information.
+	authToken    string     // Authentication token.
 }
 
 // newAuthRPCClient - returns a JWT based authenticated (go) rpc client, which does automatic reconnect.
@@ -78,33 +78,43 @@ func newAuthRPCClient(config authConfig) *AuthRPCClient {
 	}
 }
 
-// Login - a jwt based authentication is performed with rpc server.
+// Login a JWT based authentication is performed with rpc server.
 func (authClient *AuthRPCClient) Login() (err error) {
+	// Login should be attempted one at a time.
+	//
+	// The reason for large region lock here is
+	// to avoid two simultaneous login attempts
+	// racing over each other.
+	//
+	// #1 Login() gets the lock proceeds to login.
+	// #2 Login() waits for the unlock to happen
+	//    after login in #1.
+	// #1 Successfully completes login saves the
+	//    newly acquired token.
+	// #2 Successfully gets the lock and proceeds,
+	//    but since we have acquired the token
+	//    already the call quickly returns.
 	authClient.Lock()
 	defer authClient.Unlock()
 
-	// Return if already logged in.
-	if authClient.authToken != "" {
-		return nil
+	// Attempt to login if not logged in already.
+	if authClient.authToken == "" {
+		// Login to authenticate and acquire a new auth token.
+		var (
+			loginMethod = authClient.config.serviceName + loginMethodName
+			loginArgs   = LoginRPCArgs{
+				Username:    authClient.config.accessKey,
+				Password:    authClient.config.secretKey,
+				Version:     Version,
+				RequestTime: UTCNow(),
+			}
+			loginReply = LoginRPCReply{}
+		)
+		if err = authClient.rpcClient.Call(loginMethod, &loginArgs, &loginReply); err != nil {
+			return err
+		}
+		authClient.authToken = loginReply.AuthToken
 	}
-
-	// Call login.
-	args := LoginRPCArgs{
-		Username:    authClient.config.accessKey,
-		Password:    authClient.config.secretKey,
-		Version:     Version,
-		RequestTime: UTCNow(),
-	}
-
-	reply := LoginRPCReply{}
-	serviceMethod := authClient.config.serviceName + loginMethodName
-	if err = authClient.rpcClient.Call(serviceMethod, &args, &reply); err != nil {
-		return err
-	}
-
-	// Logged in successfully.
-	authClient.authToken = reply.AuthToken
-
 	return nil
 }
 
@@ -112,17 +122,17 @@ func (authClient *AuthRPCClient) Login() (err error) {
 func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
 	SetAuthToken(authToken string)
 }, reply interface{}) (err error) {
-	// On successful login, execute RPC call.
-	if err = authClient.Login(); err == nil {
-		authClient.Lock()
-		// Set token and timestamp before the rpc call.
-		args.SetAuthToken(authClient.authToken)
-		authClient.Unlock()
+	if err = authClient.Login(); err != nil {
+		return err
+	} // On successful login, execute RPC call.
 
-		// Do RPC call.
-		err = authClient.rpcClient.Call(serviceMethod, args, reply)
-	}
-	return err
+	authClient.RLock()
+	// Set token before the rpc call.
+	args.SetAuthToken(authClient.authToken)
+	authClient.RUnlock()
+
+	// Do an RPC call.
+	return authClient.rpcClient.Call(serviceMethod, args, reply)
 }
 
 // Call executes RPC call till success or globalAuthRPCRetryThreshold on ErrShutdown.

cmd/auth-rpc-server_test.go

@@ -77,7 +77,7 @@ func TestLogin(t *testing.T) {
 		// Invalid password length
 		{
 			args: LoginRPCArgs{
-				Username: globalMinioDefaultOwnerID,
+				Username: "minio",
 				Password: "aaa",
 				Version:  Version,
 			},

cmd/benchmark-utils_test.go

@@ -39,7 +39,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -49,7 +49,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	metadata := make(map[string]string)
-	metadata["md5Sum"] = getMD5Hash(textData)
+	metadata["etag"] = getMD5Hash(textData)
 	sha256sum := ""
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
@@ -61,8 +61,8 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.MD5Sum != metadata["md5Sum"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		if objInfo.ETag != metadata["etag"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
 		}
 	}
 	// Benchmark ends here. Stop timer.
@@ -78,22 +78,22 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
 
 	objSize := 128 * humanize.MiByte
 
-	// PutObjectPart returns md5Sum of the object inserted.
-	// md5Sum variable is assigned with that value.
-	var md5Sum, uploadID string
+	// PutObjectPart returns etag of the object inserted.
+	// etag variable is assigned with that value.
+	var etag, uploadID string
 	// get text data generated for number of bytes equal to object size.
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
 	metadata := make(map[string]string)
-	metadata["md5Sum"] = getMD5Hash(textData)
+	metadata["etag"] = getMD5Hash(textData)
 	sha256sum := ""
 	uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
 	if err != nil {
@@ -115,14 +115,14 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 				textPartData = textData[j*partSize:]
 			}
 			metadata := make(map[string]string)
-			metadata["md5Sum"] = getMD5Hash([]byte(textPartData))
+			metadata["etag"] = getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
-			partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"], sha256sum)
+			partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["etag"], sha256sum)
 			if err != nil {
 				b.Fatal(err)
 			}
-			if partInfo.ETag != metadata["md5Sum"] {
-				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+			if partInfo.ETag != metadata["etag"] {
+				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, metadata["etag"])
 			}
 		}
 	}
@@ -199,7 +199,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -208,19 +208,19 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < 10; i++ {
 		// get text data generated for number of bytes equal to object size.
 		textData := generateBytesData(objSize)
-		// generate md5sum for the generated data.
-		// md5sum of the data to written is required as input for PutObject.
+		// generate etag for the generated data.
+		// etag of the data to written is required as input for PutObject.
 		// PutObject is the functions which writes the data onto the FS/XL backend.
 		metadata := make(map[string]string)
-		metadata["md5Sum"] = getMD5Hash(textData)
+		metadata["etag"] = getMD5Hash(textData)
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.MD5Sum != metadata["md5Sum"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		if objInfo.ETag != metadata["etag"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
 		}
 	}
 
@@ -307,7 +307,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -317,7 +317,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	metadata := make(map[string]string)
-	metadata["md5Sum"] = getMD5Hash([]byte(textData))
+	metadata["etag"] = getMD5Hash([]byte(textData))
 	sha256sum := ""
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
@@ -332,8 +332,8 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 			if err != nil {
 				b.Fatal(err)
 			}
-			if objInfo.MD5Sum != metadata["md5Sum"] {
-				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.MD5Sum, metadata["md5Sum"])
+			if objInfo.ETag != metadata["etag"] {
+				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, metadata["etag"])
 			}
 			i++
 		}
@@ -355,7 +355,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -367,7 +367,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// md5sum of the data to written is required as input for PutObject.
 		// PutObject is the functions which writes the data onto the FS/XL backend.
 		metadata := make(map[string]string)
-		metadata["md5Sum"] = getMD5Hash([]byte(textData))
+		metadata["etag"] = getMD5Hash([]byte(textData))
 		sha256sum := ""
 		// insert the object.
 		var objInfo ObjectInfo
@@ -375,8 +375,8 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.MD5Sum != metadata["md5Sum"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		if objInfo.ETag != metadata["etag"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
 		}
 	}
 

cmd/browser-rpc-router.go

@@ -17,8 +17,6 @@
 package cmd
 
 import (
-	"net/rpc"
-
 	router "github.com/gorilla/mux"
 )
 
@@ -39,7 +37,7 @@ type browserPeerAPIHandlers struct {
 func registerBrowserPeerRPCRouter(mux *router.Router) error {
 	bpHandlers := &browserPeerAPIHandlers{}
 
-	bpRPCServer := rpc.NewServer()
+	bpRPCServer := newRPCServer()
 	err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)
 	if err != nil {
 		return traceError(err)

cmd/bucket-handlers-listobjects.go

@@ -100,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, fetchOwner, maxKeys, listObjectsInfo)
+	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsInfo.NextMarker, startAfter, delimiter, fetchOwner, listObjectsInfo.IsTruncated, maxKeys, listObjectsInfo.Objects, listObjectsInfo.Prefixes)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))

cmd/bucket-handlers.go

@@ -83,6 +83,10 @@ func enforceBucketPolicy(bucket, action, resource, referer string, queryParams u
 
 // Check if the action is allowed on the bucket/prefix.
 func isBucketActionAllowed(action, bucket, prefix string) bool {
+	if globalBucketPolicies == nil {
+		return false
+	}
+
 	policy := globalBucketPolicies.GetBucketPolicy(bucket)
 	if policy == nil {
 		return false
@@ -389,7 +393,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
-	err := objectAPI.MakeBucket(bucket)
+	err := objectAPI.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		errorIf(err, "Unable to create a bucket.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -521,7 +525,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	// Extract metadata to be saved from received Form.
-	metadata := extractMetadataFromForm(formValues)
+	metadata, err := extractMetadataFromHeader(formValues)
+	if err != nil {
+		errorIf(err, "found invalid http request header")
+		writeErrorResponse(w, ErrInternalError, r.URL)
+		return
+	}
 	sha256sum := ""
 
 	objectLock := globalNSMutex.NewNSLock(bucket, object)
@@ -535,7 +544,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	w.Header().Set("ETag", `"`+objInfo.MD5Sum+`"`)
+	w.Header().Set("ETag", `"`+objInfo.ETag+`"`)
 	w.Header().Set("Location", getObjectLocation(bucket, object))
 
 	// Get host and port from Request.RemoteAddr.
@@ -568,7 +577,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		resp := encodeResponse(PostResponse{
 			Bucket:   objInfo.Bucket,
 			Key:      objInfo.Name,
-			ETag:     `"` + objInfo.MD5Sum + `"`,
+			ETag:     `"` + objInfo.ETag + `"`,
 			Location: getObjectLocation(objInfo.Bucket, objInfo.Name),
 		})
 		writeResponse(w, http.StatusCreated, resp, "application/xml")
@@ -644,12 +653,21 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	// Delete bucket access policy, if present - ignore any errors.
 	_ = removeBucketPolicy(bucket, objectAPI)
 
+	// Notify all peers (including self) to update in-memory state
+	S3PeersUpdateBucketPolicy(bucket, policyChange{true, nil})
+
 	// Delete notification config, if present - ignore any errors.
 	_ = removeNotificationConfig(bucket, objectAPI)
 
+	// Notify all peers (including self) to update in-memory state
+	S3PeersUpdateBucketNotification(bucket, nil)
+
 	// Delete listener config, if present - ignore any errors.
 	_ = removeListenerConfig(bucket, objectAPI)
 
+	// Notify all peers (including self) to update in-memory state
+	S3PeersUpdateBucketListener(bucket, []listenerConfig{})
+
 	// Write success response.
 	writeSuccessNoContent(w)
 }

cmd/bucket-handlers_test.go

@@ -68,7 +68,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 			locationResponse:   []byte(""),
 			errorResponse: APIErrorResponse{
 				Resource: "/" + bucketName + "/",
-				Code:     "InvalidAccessKeyID",
+				Code:     "InvalidAccessKeyId",
 				Message:  "The access key ID you provided does not exist in our records.",
 			},
 			shouldPass: false,
@@ -771,3 +771,36 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	// `ExecObjectLayerAPINilTest` manages the operation.
 	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
 }
+
+func TestIsBucketActionAllowed(t *testing.T) {
+	ExecObjectLayerAPITest(t, testIsBucketActionAllowedHandler, []string{"BucketLocation"})
+}
+
+func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+
+	testCases := []struct {
+		// input.
+		action              string
+		bucket              string
+		prefix              string
+		isGlobalPoliciesNil bool
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		{"s3:GetBucketLocation", "mybucket", "abc", true, false},
+		{"s3:ListObject", "mybucket", "abc", false, false},
+	}
+	for i, testCase := range testCases {
+		if testCase.isGlobalPoliciesNil {
+			globalBucketPolicies = nil
+		} else {
+			initBucketPolicies(obj)
+		}
+		isAllowed := isBucketActionAllowed(testCase.action, testCase.bucket, testCase.prefix)
+		if isAllowed != testCase.shouldPass {
+			t.Errorf("Case %d: Expected the response status to be `%t`, but instead found `%t`", i+1, testCase.shouldPass, isAllowed)
+		}
+
+	}
+}

cmd/bucket-notification-handlers.go

@@ -204,6 +204,15 @@ func writeNotification(w http.ResponseWriter, notification map[string][]Notifica
 	if err != nil {
 		return err
 	}
+
+	// https://github.com/containous/traefik/issues/560
+	// https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
+	//
+	// Proxies might buffer the connection to avoid this we
+	// need the proper MIME type before writing to client.
+	// This MIME header tells the proxies to avoid buffering
+	w.Header().Set("Content-Type", "text/event-stream")
+
 	// Add additional CRLF characters for client to
 	// differentiate the individual events properly.
 	_, err = w.Write(append(notificationBytes, crlf...))

cmd/bucket-notification-handlers_test.go

@@ -185,7 +185,7 @@ func testGetBucketNotificationHandler(obj ObjectLayer, instanceType, bucketName
 	filterRules := []filterRule{
 		{
 			Name:  "prefix",
-			Value: globalMinioDefaultOwnerID,
+			Value: "minio",
 		},
 		{
 			Name:  "suffix",

cmd/bucket-notification-utils.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"errors"
 	"strings"
 
 	"github.com/minio/minio-go/pkg/set"
@@ -111,20 +112,21 @@ func checkARN(arn, arnType string) APIErrorCode {
 	if !strings.HasPrefix(arn, arnType) {
 		return ErrARNNotification
 	}
-	if !strings.HasPrefix(arn, arnType+serverConfig.GetRegion()+":") {
-		return ErrRegionNotification
-	}
-	account := strings.SplitN(strings.TrimPrefix(arn, arnType+serverConfig.GetRegion()+":"), ":", 2)
-	switch len(account) {
-	case 1:
-		// This means ARN is malformed, account should have min of 2elements.
+	strs := strings.SplitN(arn, ":", -1)
+	if len(strs) != 6 {
 		return ErrARNNotification
-	case 2:
-		// Account topic id or topic name cannot be empty.
-		if account[0] == "" || account[1] == "" {
-			return ErrARNNotification
+	}
+	if serverConfig.GetRegion() != "" {
+		region := strs[3]
+		if region != serverConfig.GetRegion() {
+			return ErrRegionNotification
 		}
 	}
+	accountID := strs[4]
+	resource := strs[5]
+	if accountID == "" || resource == "" {
+		return ErrARNNotification
+	}
 	return ErrNone
 }
 
@@ -142,6 +144,9 @@ func isValidQueueID(queueARN string) bool {
 	if isAMQPQueue(sqsARN) { // AMQP eueue.
 		amqpN := serverConfig.Notify.GetAMQPByID(sqsARN.AccountID)
 		return amqpN.Enable && amqpN.URL != ""
+	} else if isMQTTQueue(sqsARN) {
+		mqttN := serverConfig.Notify.GetMQTTByID(sqsARN.AccountID)
+		return mqttN.Enable && mqttN.Broker != ""
 	} else if isNATSQueue(sqsARN) {
 		natsN := serverConfig.Notify.GetNATSByID(sqsARN.AccountID)
 		return natsN.Enable && natsN.Address != ""
@@ -249,6 +254,7 @@ func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
 // Unmarshals input value of AWS ARN format into minioSqs object.
 // Returned value represents minio sqs types, currently supported are
 // - amqp
+// - mqtt
 // - nats
 // - elasticsearch
 // - redis
@@ -257,29 +263,41 @@ func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
 // - kafka
 // - webhook
 func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
-	mSqs = arnSQS{}
-	if !strings.HasPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":") {
-		return mSqs
+	strs := strings.SplitN(queueARN, ":", -1)
+	if len(strs) != 6 {
+		return
 	}
-	sqsType := strings.TrimPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":")
-	switch {
-	case hasSuffix(sqsType, queueTypeAMQP):
+	if serverConfig.GetRegion() != "" {
+		region := strs[3]
+		if region != serverConfig.GetRegion() {
+			return
+		}
+	}
+	sqsType := strs[5]
+	switch sqsType {
+	case queueTypeAMQP:
 		mSqs.Type = queueTypeAMQP
-	case hasSuffix(sqsType, queueTypeNATS):
+	case queueTypeMQTT:
+		mSqs.Type = queueTypeMQTT
+	case queueTypeNATS:
 		mSqs.Type = queueTypeNATS
-	case hasSuffix(sqsType, queueTypeElastic):
+	case queueTypeElastic:
 		mSqs.Type = queueTypeElastic
-	case hasSuffix(sqsType, queueTypeRedis):
+	case queueTypeRedis:
 		mSqs.Type = queueTypeRedis
-	case hasSuffix(sqsType, queueTypePostgreSQL):
+	case queueTypePostgreSQL:
 		mSqs.Type = queueTypePostgreSQL
-	case hasSuffix(sqsType, queueTypeMySQL):
+	case queueTypeMySQL:
 		mSqs.Type = queueTypeMySQL
-	case hasSuffix(sqsType, queueTypeKafka):
+	case queueTypeKafka:
 		mSqs.Type = queueTypeKafka
-	case hasSuffix(sqsType, queueTypeWebhook):
+	case queueTypeWebhook:
 		mSqs.Type = queueTypeWebhook
+	default:
+		errorIf(errors.New("invalid SQS type"), "SQS type: %s", sqsType)
 	} // Add more queues here.
-	mSqs.AccountID = strings.TrimSuffix(sqsType, ":"+mSqs.Type)
-	return mSqs
+
+	mSqs.AccountID = strs[4]
+
+	return
 }

cmd/bucket-notification-utils_test.go

@@ -259,11 +259,6 @@ func TestQueueARN(t *testing.T) {
 			queueARN: "arn:minio:sns:us-east-1:1:listen",
 			errCode:  ErrARNNotification,
 		},
-		// Invalid region 'us-west-1' in queue arn.
-		{
-			queueARN: "arn:minio:sqs:us-west-1:1:redis",
-			errCode:  ErrRegionNotification,
-		},
 		// Invalid queue name empty in queue arn.
 		{
 			queueARN: "arn:minio:sqs:us-east-1:1:",
@@ -298,6 +293,37 @@ func TestQueueARN(t *testing.T) {
 			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
 		}
 	}
+
+	// Test when server region is set.
+	rootPath, err = newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(rootPath)
+
+	testCases = []struct {
+		queueARN string
+		errCode  APIErrorCode
+	}{
+		// Incorrect region should produce error.
+		{
+			queueARN: "arn:minio:sqs:us-west-1:1:webhook",
+			errCode:  ErrRegionNotification,
+		},
+		// Correct region should not produce error.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:webhook",
+			errCode:  ErrNone,
+		},
+	}
+
+	// Validate all tests for queue arn.
+	for i, testCase := range testCases {
+		errCode := checkQueueARN(testCase.queueARN)
+		if testCase.errCode != errCode {
+			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
+		}
+	}
 }
 
 // Test unmarshal queue arn.
@@ -332,16 +358,16 @@ func TestUnmarshalSQSARN(t *testing.T) {
 			queueARN: "arn:minio:sqs:us-east-1:1:amqp",
 			Type:     "amqp",
 		},
+		// Valid mqtt queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:mqtt",
+			Type:     "mqtt",
+		},
 		// Invalid empty queue arn.
 		{
 			queueARN: "",
 			Type:     "",
 		},
-		// Invalid region 'us-west-1' in queue arn.
-		{
-			queueARN: "arn:minio:sqs:us-west-1:1:redis",
-			Type:     "",
-		},
 		// Partial queue arn.
 		{
 			queueARN: "arn:minio:sqs:",
@@ -361,4 +387,33 @@ func TestUnmarshalSQSARN(t *testing.T) {
 		}
 	}
 
+	// Test when the server region is set.
+	rootPath, err = newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(rootPath)
+
+	testCases = []struct {
+		queueARN string
+		Type     string
+	}{
+		// Incorrect region in ARN returns empty mSqs.Type
+		{
+			queueARN: "arn:minio:sqs:us-west-1:1:webhook",
+			Type:     "",
+		},
+		// Correct regionin ARN returns valid mSqs.Type
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:webhook",
+			Type:     "webhook",
+		},
+	}
+
+	for i, testCase := range testCases {
+		mSqs := unmarshalSqsARN(testCase.queueARN)
+		if testCase.Type != mSqs.Type {
+			t.Errorf("Test %d: Expected \"%s\", got \"%s\"", i+1, testCase.Type, mSqs.Type)
+		}
+	}
 }

cmd/bucket-policy-handlers_test.go

@@ -251,7 +251,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	initBucketPolicies(obj)
 
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucket(bucketName1); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName1, ""); err != nil {
 		t.Fatal(err)
 	}
 

cmd/certs.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
@@ -24,35 +25,34 @@ import (
 	"path/filepath"
 )
 
-func parsePublicCertFile(certFile string) (certs []*x509.Certificate, err error) {
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(certFile); err != nil {
-		return certs, err
+func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
+	// Read certificate file.
+	var data []byte
+	if data, err = ioutil.ReadFile(certFile); err != nil {
+		return nil, err
 	}
 
 	// Parse all certs in the chain.
-	var block *pem.Block
-	var cert *x509.Certificate
-	current := bytes
+	current := data
 	for len(current) > 0 {
-		if block, current = pem.Decode(current); block == nil {
-			err = fmt.Errorf("Could not read PEM block from file %s", certFile)
-			return certs, err
+		var pemBlock *pem.Block
+		if pemBlock, current = pem.Decode(current); pemBlock == nil {
+			return nil, fmt.Errorf("Could not read PEM block from file %s", certFile)
 		}
 
-		if cert, err = x509.ParseCertificate(block.Bytes); err != nil {
-			return certs, err
+		var x509Cert *x509.Certificate
+		if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
+			return nil, err
 		}
 
-		certs = append(certs, cert)
+		x509Certs = append(x509Certs, x509Cert)
 	}
 
-	if len(certs) == 0 {
-		err = fmt.Errorf("Empty public certificate file %s", certFile)
+	if len(x509Certs) == 0 {
+		return nil, fmt.Errorf("Empty public certificate file %s", certFile)
 	}
 
-	return certs, err
+	return x509Certs, nil
 }
 
 func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
@@ -81,7 +81,7 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	for _, caFile := range caFiles {
 		caCert, err := ioutil.ReadFile(caFile)
 		if err != nil {
-			return rootCAs, err
+			return nil, err
 		}
 
 		rootCAs.AppendCertsFromPEM(caCert)
@@ -90,19 +90,26 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	return rootCAs, nil
 }
 
-func getSSLConfig() (publicCerts []*x509.Certificate, rootCAs *x509.CertPool, secureConn bool, err error) {
+func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
-		return publicCerts, rootCAs, secureConn, err
+		return nil, nil, nil, false, nil
 	}
 
-	if publicCerts, err = parsePublicCertFile(getPublicCertFile()); err != nil {
-		return publicCerts, rootCAs, secureConn, err
+	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
+		return nil, nil, nil, false, err
 	}
 
+	var cert tls.Certificate
+	if cert, err = tls.LoadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
+		return nil, nil, nil, false, err
+	}
+
+	tlsCert = &cert
+
 	if rootCAs, err = getRootCAs(getCADir()); err != nil {
-		return publicCerts, rootCAs, secureConn, err
+		return nil, nil, nil, false, err
 	}
 
 	secureConn = true
-	return publicCerts, rootCAs, secureConn, err
+	return x509Certs, rootCAs, tlsCert, secureConn, nil
 }

cmd/common-main.go

@@ -0,0 +1,117 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/minio/cli"
+)
+
+// Check for updates and print a notification message
+func checkUpdate(mode string) {
+	// Its OK to ignore any errors during getUpdateInfo() here.
+	if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil {
+		if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" {
+			log.Println(updateMsg)
+		}
+	}
+}
+
+func enableLoggers() {
+	fileLogTarget := serverConfig.Logger.GetFile()
+	if fileLogTarget.Enable {
+		err := InitFileLogger(&fileLogTarget)
+		fatalIf(err, "Unable to initialize file logger")
+		log.AddTarget(fileLogTarget)
+	}
+
+	consoleLogTarget := serverConfig.Logger.GetConsole()
+	if consoleLogTarget.Enable {
+		InitConsoleLogger(&consoleLogTarget)
+	}
+
+	log.SetConsoleTarget(consoleLogTarget)
+}
+
+func initConfig() {
+	// Config file does not exist, we create it fresh and return upon success.
+	if isFile(getConfigFile()) {
+		fatalIf(migrateConfig(), "Config migration failed.")
+		fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19)
+	} else {
+		fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
+		log.Println("Created minio configuration file successfully at " + getConfigDir())
+	}
+}
+
+func handleCommonCmdArgs(ctx *cli.Context) {
+	// Set configuration directory.
+	{
+		// Get configuration directory from command line argument.
+		configDir := ctx.String("config-dir")
+		if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
+			configDir = ctx.GlobalString("config-dir")
+		}
+		if configDir == "" {
+			fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
+		}
+
+		// Disallow relative paths, figure out absolute paths.
+		configDirAbs, err := filepath.Abs(configDir)
+		fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
+
+		setConfigDir(configDirAbs)
+	}
+}
+
+func handleCommonEnvVars() {
+	// Start profiler if env is set.
+	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
+		globalProfiler = startProfiler(profiler)
+	}
+
+	// Check if object cache is disabled.
+	globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
+
+	accessKey := os.Getenv("MINIO_ACCESS_KEY")
+	secretKey := os.Getenv("MINIO_SECRET_KEY")
+	if accessKey != "" && secretKey != "" {
+		cred, err := createCredential(accessKey, secretKey)
+		fatalIf(err, "Invalid access/secret Key set in environment.")
+
+		// credential Envs are set globally.
+		globalIsEnvCreds = true
+		globalActiveCred = cred
+	}
+
+	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
+		browserFlag, err := ParseBrowserFlag(browser)
+		if err != nil {
+			fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
+		}
+
+		// browser Envs are set globally, this does not represent
+		// if browser is turned off or on.
+		globalIsEnvBrowser = true
+		globalIsBrowserEnabled = bool(browserFlag)
+	}
+}

cmd/config-migrate.go

@@ -27,77 +27,131 @@ import (
 // DO NOT EDIT following message template, please open a github issue to discuss instead.
 var configMigrateMSGTemplate = "Configuration file %s migrated from version '%s' to '%s' successfully.\n"
 
+// Migrates all config versions from "1" to "18".
 func migrateConfig() error {
-	// Purge all configs with version '1'.
+	// Purge all configs with version '1',
+	// this is a special case since version '1' used
+	// to be a filename 'fsUsers.json' not 'config.json'.
 	if err := purgeV1(); err != nil {
 		return err
 	}
-	// Migrate version '2' to '3'.
-	if err := migrateV2ToV3(); err != nil {
-		return err
-	}
-	// Migrate version '3' to '4'.
-	if err := migrateV3ToV4(); err != nil {
-		return err
-	}
-	// Migrate version '4' to '5'.
-	if err := migrateV4ToV5(); err != nil {
-		return err
-	}
-	// Migrate version '5' to '6.
-	if err := migrateV5ToV6(); err != nil {
-		return err
-	}
-	// Migrate version '6' to '7'.
-	if err := migrateV6ToV7(); err != nil {
-		return err
-	}
-	// Migrate version '7' to '8'.
-	if err := migrateV7ToV8(); err != nil {
-		return err
-	}
-	// Migrate version '8' to '9'.
-	if err := migrateV8ToV9(); err != nil {
-		return err
-	}
-	// Migrate version '9' to '10'.
-	if err := migrateV9ToV10(); err != nil {
-		return err
-	}
-	// Migrate version '10' to '11'.
-	if err := migrateV10ToV11(); err != nil {
-		return err
-	}
-	// Migrate version '11' to '12'.
-	if err := migrateV11ToV12(); err != nil {
-		return err
-	}
-	// Migrate version '12' to '13'.
-	if err := migrateV12ToV13(); err != nil {
-		return err
-	}
-	// Migrate version '13' to '14'.
-	if err := migrateV13ToV14(); err != nil {
-		return err
-	}
-	// Migrate version '14' to '15'.
-	if err := migrateV14ToV15(); err != nil {
-		return err
-	}
-	// Migrate version '15' to '16'.
-	if err := migrateV15ToV16(); err != nil {
-		return err
-	}
-	// Migrate version '16' to '17'.
-	if err := migrateV16ToV17(); err != nil {
-		return err
-	}
-	// Migrate version '17' to '18'.
-	if err := migrateV17ToV18(); err != nil {
+
+	// Load only config version information.
+	version, err := quick.GetVersion(getConfigFile())
+	if err != nil {
 		return err
 	}
 
-	return nil
+	// Conditional to migrate only relevant config versions.
+	// Upon success migration continues to the next version in sequence.
+	switch version {
+	case "2":
+		// Migrate version '2' to '3'.
+		if err = migrateV2ToV3(); err != nil {
+			return err
+		}
+		fallthrough
+	case "3":
+		// Migrate version '3' to '4'.
+		if err = migrateV3ToV4(); err != nil {
+			return err
+		}
+		fallthrough
+	case "4":
+		// Migrate version '4' to '5'.
+		if err = migrateV4ToV5(); err != nil {
+			return err
+		}
+		fallthrough
+	case "5":
+		// Migrate version '5' to '6.
+		if err = migrateV5ToV6(); err != nil {
+			return err
+		}
+		fallthrough
+	case "6":
+		// Migrate version '6' to '7'.
+		if err = migrateV6ToV7(); err != nil {
+			return err
+		}
+		fallthrough
+	case "7":
+		// Migrate version '7' to '8'.
+		if err = migrateV7ToV8(); err != nil {
+			return err
+		}
+		fallthrough
+	case "8":
+		// Migrate version '8' to '9'.
+		if err = migrateV8ToV9(); err != nil {
+			return err
+		}
+		fallthrough
+	case "9":
+		// Migrate version '9' to '10'.
+		if err = migrateV9ToV10(); err != nil {
+			return err
+		}
+		fallthrough
+	case "10":
+		// Migrate version '10' to '11'.
+		if err = migrateV10ToV11(); err != nil {
+			return err
+		}
+		fallthrough
+	case "11":
+		// Migrate version '11' to '12'.
+		if err = migrateV11ToV12(); err != nil {
+			return err
+		}
+		fallthrough
+	case "12":
+		// Migrate version '12' to '13'.
+		if err = migrateV12ToV13(); err != nil {
+			return err
+		}
+		fallthrough
+	case "13":
+		// Migrate version '13' to '14'.
+		if err = migrateV13ToV14(); err != nil {
+			return err
+		}
+		fallthrough
+	case "14":
+		// Migrate version '14' to '15'.
+		if err = migrateV14ToV15(); err != nil {
+			return err
+		}
+		fallthrough
+	case "15":
+		// Migrate version '15' to '16'.
+		if err = migrateV15ToV16(); err != nil {
+			return err
+		}
+		fallthrough
+	case "16":
+		// Migrate version '16' to '17'.
+		if err = migrateV16ToV17(); err != nil {
+			return err
+		}
+		fallthrough
+	case "17":
+		// Migrate version '17' to '18'.
+		if err = migrateV17ToV18(); err != nil {
+			return err
+		}
+		fallthrough
+	case "18":
+		// Migrate version '17' to '18'.
+		if err = migrateV18ToV19(); err != nil {
+			return err
+		}
+		fallthrough
+	case v19:
+		// No migration needed. this always points to current version.
+		err = nil
+	}
+	return err
 }
 
 // Version '1' is not supported anymore and deprecated, safe to delete.
@@ -1318,3 +1372,109 @@ func migrateV17ToV18() error {
 	log.Printf(configMigrateMSGTemplate, configFile, cv17.Version, srvConfig.Version)
 	return nil
 }
+
+func migrateV18ToV19() error {
+	configFile := getConfigFile()
+
+	cv18 := &serverConfigV18{}
+	_, err := quick.Load(configFile, cv18)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
+	}
+	if cv18.Version != "18" {
+		return nil
+	}
+
+	// Copy over fields from V18 into V19 config struct
+	srvConfig := &serverConfigV18{
+		Logger: &loggers{},
+		Notify: &notifier{},
+	}
+	srvConfig.Version = "19"
+	srvConfig.Credential = cv18.Credential
+	srvConfig.Region = cv18.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	srvConfig.Logger.Console = cv18.Logger.Console
+	srvConfig.Logger.File = cv18.Logger.File
+
+	// check and set notifiers config
+	if len(cv18.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		// New deliveryMode parameter is added for AMQP,
+		// default value is already 0, so nothing to
+		// explicitly migrate here.
+		srvConfig.Notify.AMQP = cv18.Notify.AMQP
+	}
+	if len(cv18.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv18.Notify.ElasticSearch
+	}
+	if len(cv18.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]redisNotify)
+		srvConfig.Notify.Redis["1"] = redisNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv18.Notify.Redis
+	}
+	if len(cv18.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
+		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv18.Notify.PostgreSQL
+	}
+	if len(cv18.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
+		srvConfig.Notify.Kafka["1"] = kafkaNotify{}
+	} else {
+		srvConfig.Notify.Kafka = cv18.Notify.Kafka
+	}
+	if len(cv18.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]natsNotify)
+		srvConfig.Notify.NATS["1"] = natsNotify{}
+	} else {
+		srvConfig.Notify.NATS = cv18.Notify.NATS
+	}
+	if len(cv18.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]webhookNotify)
+		srvConfig.Notify.Webhook["1"] = webhookNotify{}
+	} else {
+		srvConfig.Notify.Webhook = cv18.Notify.Webhook
+	}
+	if len(cv18.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
+		srvConfig.Notify.MySQL["1"] = mySQLNotify{
+			Format: formatNamespace,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv18.Notify.MySQL
+	}
+
+	// V18 will not have mqtt support, so we add that here.
+	srvConfig.Notify.MQTT = make(map[string]mqttNotify)
+	srvConfig.Notify.MQTT["1"] = mqttNotify{}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv18.Browser
+
+	if err = quick.Save(configFile, srvConfig); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv18.Version, srvConfig.Version, err)
+	}
+
+	log.Printf(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version)
+	return nil
+}

cmd/config-migrate_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -121,11 +122,14 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	if err := migrateV17ToV18(); err != nil {
 		t.Fatal("migrate v17 to v18 should succeed when no config file is found")
 	}
+	if err := migrateV18ToV19(); err != nil {
+		t.Fatal("migrate v18 to v19 should succeed when no config file is found")
+	}
 
 }
 
-// Test if a config migration from v2 to v18 is successfully done
-func TestServerConfigMigrateV2toV18(t *testing.T) {
+// Test if a config migration from v2 to v19 is successfully done
+func TestServerConfigMigrateV2toV19(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Init Test config failed")
@@ -153,6 +157,7 @@ func TestServerConfigMigrateV2toV18(t *testing.T) {
 	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
+
 	// Fire a migrateConfig()
 	if err := migrateConfig(); err != nil {
 		t.Fatal("Unexpected error: ", err)
@@ -164,7 +169,7 @@ func TestServerConfigMigrateV2toV18(t *testing.T) {
 	}
 
 	// Check the version number in the upgraded config file
-	expectedVersion := v18
+	expectedVersion := v19
 	if serverConfig.Version != expectedVersion {
 		t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
 	}
@@ -191,7 +196,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	configPath := rootPath + "/" + minioConfigFile
 
 	// Create a corrupted config file
-	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\""), 0644); err != nil {
+	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -244,4 +249,43 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV17ToV18(); err == nil {
 		t.Fatal("migrateConfigV17ToV18() should fail with a corrupted json")
 	}
+	if err := migrateV18ToV19(); err == nil {
+		t.Fatal("migrateConfigV18ToV19() should fail with a corrupted json")
+	}
+}
+
+// Test if all migrate code returns error with corrupted config files
+func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
+	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	setConfigDir(rootPath)
+	configPath := rootPath + "/" + minioConfigFile
+
+	for i := 3; i <= 17; i++ {
+		// Create a corrupted config file
+		if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
+			0644); err != nil {
+			t.Fatal("Unexpected error: ", err)
+		}
+
+		// Test different migrate versions and be sure they are returning an error
+		if err = migrateConfig(); err == nil {
+			t.Fatal("migrateConfig() should fail with a corrupted json")
+		}
+	}
+
+	// Create a corrupted config file for version '2'.
+	if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0644); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
+	// Test different migrate versions and be sure they are returning an error
+	if err = migrateConfig(); err == nil {
+		t.Fatal("migrateConfig() should fail with a corrupted json")
+	}
 }

cmd/config-old.go

@@ -449,3 +449,22 @@ type serverConfigV17 struct {
 	// Notification queue configuration.
 	Notify *notifier `json:"notify"`
 }
+
+// serverConfigV18 server configuration version '18' which is like
+// version '17' except it adds support for "deliveryMode" parameter in
+// the AMQP notification target.
+type serverConfigV18 struct {
+	sync.RWMutex
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}

cmd/config-v19.go

@@ -27,18 +27,17 @@ import (
 )
 
 // Config version
-const v18 = "18"
+const v19 = "19"
 
 var (
 	// serverConfig server config.
-	serverConfig   *serverConfigV18
+	serverConfig   *serverConfigV19
 	serverConfigMu sync.RWMutex
 )
 
-// serverConfigV18 server configuration version '18' which is like
-// version '17' except it adds support for "deliveryMode" parameter in
-// the AMQP notification target.
-type serverConfigV18 struct {
+// serverConfigV19 server configuration version '19' which is like
+// version '18' except it adds support for MQTT notifications.
+type serverConfigV19 struct {
 	sync.RWMutex
 	Version string `json:"version"`
 
@@ -55,7 +54,7 @@ type serverConfigV18 struct {
 }
 
 // GetVersion get current config version.
-func (s *serverConfigV18) GetVersion() string {
+func (s *serverConfigV19) GetVersion() string {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -63,7 +62,7 @@ func (s *serverConfigV18) GetVersion() string {
 }
 
 // SetRegion set new region.
-func (s *serverConfigV18) SetRegion(region string) {
+func (s *serverConfigV19) SetRegion(region string) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -71,7 +70,7 @@ func (s *serverConfigV18) SetRegion(region string) {
 }
 
 // GetRegion get current region.
-func (s *serverConfigV18) GetRegion() string {
+func (s *serverConfigV19) GetRegion() string {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -79,7 +78,7 @@ func (s *serverConfigV18) GetRegion() string {
 }
 
 // SetCredentials set new credentials.
-func (s *serverConfigV18) SetCredential(creds credential) {
+func (s *serverConfigV19) SetCredential(creds credential) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -88,7 +87,7 @@ func (s *serverConfigV18) SetCredential(creds credential) {
 }
 
 // GetCredentials get current credentials.
-func (s *serverConfigV18) GetCredential() credential {
+func (s *serverConfigV19) GetCredential() credential {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -96,7 +95,7 @@ func (s *serverConfigV18) GetCredential() credential {
 }
 
 // SetBrowser set if browser is enabled.
-func (s *serverConfigV18) SetBrowser(b bool) {
+func (s *serverConfigV19) SetBrowser(b bool) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -105,7 +104,7 @@ func (s *serverConfigV18) SetBrowser(b bool) {
 }
 
 // GetCredentials get current credentials.
-func (s *serverConfigV18) GetBrowser() bool {
+func (s *serverConfigV19) GetBrowser() bool {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -113,7 +112,7 @@ func (s *serverConfigV18) GetBrowser() bool {
 }
 
 // Save config.
-func (s *serverConfigV18) Save() error {
+func (s *serverConfigV19) Save() error {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -121,9 +120,9 @@ func (s *serverConfigV18) Save() error {
 	return quick.Save(getConfigFile(), s)
 }
 
-func newServerConfigV18() *serverConfigV18 {
-	srvCfg := &serverConfigV18{
-		Version:    v18,
+func newServerConfigV19() *serverConfigV19 {
+	srvCfg := &serverConfigV19{
+		Version:    v19,
 		Credential: mustGetNewCredential(),
 		Region:     globalMinioDefaultRegion,
 		Browser:    true,
@@ -137,6 +136,8 @@ func newServerConfigV18() *serverConfigV18 {
 	// Make sure to initialize notification configs.
 	srvCfg.Notify.AMQP = make(map[string]amqpNotify)
 	srvCfg.Notify.AMQP["1"] = amqpNotify{}
+	srvCfg.Notify.MQTT = make(map[string]mqttNotify)
+	srvCfg.Notify.MQTT["1"] = mqttNotify{}
 	srvCfg.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
 	srvCfg.Notify.ElasticSearch["1"] = elasticSearchNotify{}
 	srvCfg.Notify.Redis = make(map[string]redisNotify)
@@ -159,7 +160,7 @@ func newServerConfigV18() *serverConfigV18 {
 // found, otherwise use default parameters
 func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfigV18()
+	srvCfg := newServerConfigV19()
 
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
@@ -237,8 +238,8 @@ func checkDupJSONKeys(json string) error {
 }
 
 // getValidConfig - returns valid server configuration
-func getValidConfig() (*serverConfigV18, error) {
-	srvCfg := &serverConfigV18{
+func getValidConfig() (*serverConfigV19, error) {
+	srvCfg := &serverConfigV19{
 		Region:  globalMinioDefaultRegion,
 		Browser: true,
 	}
@@ -248,8 +249,8 @@ func getValidConfig() (*serverConfigV18, error) {
 		return nil, err
 	}
 
-	if srvCfg.Version != v18 {
-		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v18, srvCfg.Version)
+	if srvCfg.Version != v19 {
+		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v19, srvCfg.Version)
 	}
 
 	// Load config file json and check for duplication json keys
@@ -261,11 +262,6 @@ func getValidConfig() (*serverConfigV18, error) {
 		return nil, err
 	}
 
-	// Validate region field
-	if srvCfg.Region == "" {
-		return nil, errors.New("Region config value cannot be empty")
-	}
-
 	// Validate credential fields only when
 	// they are not set via the environment
 

cmd/config-v19_test.go

@@ -80,13 +80,21 @@ func TestServerConfig(t *testing.T) {
 	}
 
 	// Set new console logger.
-	// Set new Webhook notification id.
+	// Set new MySQL notification id.
 	serverConfig.Notify.SetMySQLByID("2", mySQLNotify{})
 	savedNotifyCfg6 := serverConfig.Notify.GetMySQLByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg6, mySQLNotify{}) {
 		t.Errorf("Expecting Webhook config %#v found %#v", mySQLNotify{}, savedNotifyCfg6)
 	}
 
+	// Set new console logger.
+	// Set new MQTT notification id.
+	serverConfig.Notify.SetMQTTByID("2", mqttNotify{})
+	savedNotifyCfg7 := serverConfig.Notify.GetMQTTByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg7, mqttNotify{}) {
+		t.Errorf("Expecting Webhook config %#v found %#v", mqttNotify{}, savedNotifyCfg7)
+	}
+
 	consoleLogger := NewConsoleLogger()
 	serverConfig.Logger.SetConsole(consoleLogger)
 	consoleCfg := serverConfig.Logger.GetConsole()
@@ -109,8 +117,8 @@ func TestServerConfig(t *testing.T) {
 	serverConfig.Logger.SetFile(fileLogger)
 
 	// Match version.
-	if serverConfig.GetVersion() != v18 {
-		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v18)
+	if serverConfig.GetVersion() != v19 {
+		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v19)
 	}
 
 	// Attempt to save.
@@ -223,7 +231,7 @@ func TestValidateConfig(t *testing.T) {
 
 	configPath := filepath.Join(rootPath, minioConfigFile)
 
-	v := v18
+	v := v19
 
 	testCases := []struct {
 		configData string
@@ -309,6 +317,9 @@ func TestValidateConfig(t *testing.T) {
 
 		// Test 28 - Test valid Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
+
+		// Test 29 - Test MQTT
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
 	}
 
 	for i, testCase := range testCases {

cmd/credential.go

@@ -25,19 +25,35 @@ import (
 )
 
 const (
-	accessKeyMinLen       = 5
-	accessKeyMaxLen       = 20
-	secretKeyMinLen       = 8
-	secretKeyMaxLenAmazon = 40
-	alphaNumericTable     = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-	alphaNumericTableLen  = byte(len(alphaNumericTable))
+	// Minimum length for Minio access key.
+	accessKeyMinLen = 5
+
+	// Maximum length for Minio access key.
+	accessKeyMaxLen = 20
+
+	// Minimum length for Minio secret key for both server and gateway mode.
+	secretKeyMinLen = 8
+
+	// Maximum secret key length for Minio, this
+	// is used when autogenerating new credentials.
+	secretKeyMaxLenMinio = 40
+
+	// Maximum secret key length allowed from client side
+	// caters for both server and gateway mode.
+	secretKeyMaxLen = 100
+
+	// Alpha numeric table used for generating access keys.
+	alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+	// Total length of the alpha numeric table.
+	alphaNumericTableLen = byte(len(alphaNumericTable))
 )
 
+// Common errors generated for access and secret key validation.
 var (
 	errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be 5 to 20 characters in length")
-	errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be 8 to 40 characters in length")
+	errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be 8 to 100 characters in length")
 )
-var secretKeyMaxLen = secretKeyMaxLenAmazon
 
 // isAccessKeyValid - validate access key for right length.
 func isAccessKeyValid(accessKey string) bool {
@@ -111,10 +127,10 @@ func mustGetNewCredential() credential {
 	accessKey := string(keyBytes)
 
 	// Generate secret key.
-	keyBytes = make([]byte, secretKeyMaxLen)
+	keyBytes = make([]byte, secretKeyMaxLenMinio)
 	_, err = rand.Read(keyBytes)
 	fatalIf(err, "Unable to generate secret key.")
-	secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLen])
+	secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLenMinio])
 
 	cred, err := createCredential(accessKey, secretKey)
 	fatalIf(err, "Unable to generate new credential.")

cmd/credential_test.go

@@ -23,6 +23,9 @@ func TestMustGetNewCredential(t *testing.T) {
 	if !cred.IsValid() {
 		t.Fatalf("Failed to get new valid credential")
 	}
+	if len(cred.SecretKey) != secretKeyMaxLenMinio {
+		t.Fatalf("Invalid length %d of the secretKey credential generated, expected %d", len(cred.SecretKey), secretKeyMaxLenMinio)
+	}
 }
 
 func TestCreateCredential(t *testing.T) {
@@ -42,7 +45,7 @@ func TestCreateCredential(t *testing.T) {
 		// Secret key too small.
 		{"myuser", "pass", false, errInvalidSecretKeyLength},
 		// Secret key too long.
-		{"myuser", "pass1234567890123456789012345678901234567", false, errInvalidSecretKeyLength},
+		{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", false, errInvalidSecretKeyLength},
 		// Success when access key contains leading/trailing spaces.
 		{" user ", cred.SecretKey, true, nil},
 		{"myuser", "mypassword", true, nil},

cmd/endpoint.go

@@ -77,14 +77,14 @@ func (endpoint Endpoint) SetHTTP() {
 }
 
 // NewEndpoint - returns new endpoint based on given arguments.
-func NewEndpoint(arg string) (Endpoint, error) {
+func NewEndpoint(arg string) (ep Endpoint, e error) {
 	// isEmptyPath - check whether given path is not empty.
 	isEmptyPath := func(path string) bool {
-		return path == "" || path == "." || path == "/" || path == `\`
+		return path == "" || path == "/" || path == `\`
 	}
 
 	if isEmptyPath(arg) {
-		return Endpoint{}, fmt.Errorf("empty or root endpoint is not supported")
+		return ep, fmt.Errorf("empty or root endpoint is not supported")
 	}
 
 	var isLocal bool
@@ -96,13 +96,13 @@ func NewEndpoint(arg string) (Endpoint, error) {
 		// - All field should be empty except Host and Path.
 		if !((u.Scheme == "http" || u.Scheme == "https") &&
 			u.User == nil && u.Opaque == "" && u.ForceQuery == false && u.RawQuery == "" && u.Fragment == "") {
-			return Endpoint{}, fmt.Errorf("invalid URL endpoint format")
+			return ep, fmt.Errorf("invalid URL endpoint format")
 		}
 
 		host, port, err := net.SplitHostPort(u.Host)
 		if err != nil {
 			if !strings.Contains(err.Error(), "missing port in address") {
-				return Endpoint{}, fmt.Errorf("invalid URL endpoint format: %s", err)
+				return ep, fmt.Errorf("invalid URL endpoint format: %s", err)
 			}
 
 			host = u.Host
@@ -110,31 +110,27 @@ func NewEndpoint(arg string) (Endpoint, error) {
 			var p int
 			p, err = strconv.Atoi(port)
 			if err != nil {
-				return Endpoint{}, fmt.Errorf("invalid URL endpoint format: invalid port number")
+				return ep, fmt.Errorf("invalid URL endpoint format: invalid port number")
 			} else if p < 1 || p > 65535 {
-				return Endpoint{}, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
+				return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
 			}
 		}
 
 		if host == "" {
-			return Endpoint{}, fmt.Errorf("invalid URL endpoint format: empty host name")
+			return ep, fmt.Errorf("invalid URL endpoint format: empty host name")
 		}
 
 		// As this is path in the URL, we should use path package, not filepath package.
 		// On MS Windows, filepath.Clean() converts into Windows path style ie `/foo` becomes `\foo`
 		u.Path = path.Clean(u.Path)
 		if isEmptyPath(u.Path) {
-			return Endpoint{}, fmt.Errorf("empty or root path is not supported in URL endpoint")
+			return ep, fmt.Errorf("empty or root path is not supported in URL endpoint")
 		}
 
-		// Get IPv4 address of the host.
-		hostIPs, err := getHostIP4(host)
+		isLocal, err = isLocalHost(host)
 		if err != nil {
-			return Endpoint{}, err
+			return ep, err
 		}
-
-		// If intersection of two IP sets is not empty, then the host is local host.
-		isLocal = !localIP4.Intersection(hostIPs).IsEmpty()
 	} else {
 		u = &url.URL{Path: path.Clean(arg)}
 		isLocal = true

cmd/endpoint_test.go

@@ -62,7 +62,6 @@ func TestNewEndpoint(t *testing.T) {
 		{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil},
 		{"http://192.168.253.200/path", Endpoint{URL: u4}, URLEndpointType, nil},
 		{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
-		{".", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{"c://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},

cmd/erasure-createfile.go

@@ -28,7 +28,9 @@ import (
 // erasureCreateFile - writes an entire stream by erasure coding to
 // all the disks, writes also calculate individual block's checksum
 // for future bit-rot protection.
-func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, allowEmpty bool, blockSize int64, dataBlocks int, parityBlocks int, algo string, writeQuorum int) (bytesWritten int64, checkSums []string, err error) {
+func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, allowEmpty bool, blockSize int64,
+	dataBlocks, parityBlocks int, algo HashAlgo, writeQuorum int) (newDisks []StorageAPI, bytesWritten int64, checkSums []string, err error) {
+
 	// Allocated blockSized buffer for reading from incoming stream.
 	buf := make([]byte, blockSize)
 
@@ -41,7 +43,7 @@ func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader
 		// FIXME: this is a bug in Golang, n == 0 and err ==
 		// io.ErrUnexpectedEOF for io.ReadFull function.
 		if n == 0 && rErr == io.ErrUnexpectedEOF {
-			return 0, nil, traceError(rErr)
+			return nil, 0, nil, traceError(rErr)
 		}
 		if rErr == io.EOF {
 			// We have reached EOF on the first byte read, io.Reader
@@ -49,28 +51,28 @@ func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader
 			// data. Will create a 0byte file instead.
 			if bytesWritten == 0 && allowEmpty {
 				blocks = make([][]byte, len(disks))
-				rErr = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum)
+				newDisks, rErr = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum)
 				if rErr != nil {
-					return 0, nil, rErr
+					return nil, 0, nil, rErr
 				}
 			} // else we have reached EOF after few reads, no need to
 			// add an additional 0bytes at the end.
 			break
 		}
 		if rErr != nil && rErr != io.ErrUnexpectedEOF {
-			return 0, nil, traceError(rErr)
+			return nil, 0, nil, traceError(rErr)
 		}
 		if n > 0 {
 			// Returns encoded blocks.
 			var enErr error
 			blocks, enErr = encodeData(buf[0:n], dataBlocks, parityBlocks)
 			if enErr != nil {
-				return 0, nil, enErr
+				return nil, 0, nil, enErr
 			}
 
 			// Write to all disks.
-			if err = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum); err != nil {
-				return 0, nil, err
+			if newDisks, err = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum); err != nil {
+				return nil, 0, nil, err
 			}
 			bytesWritten += int64(n)
 		}
@@ -80,7 +82,7 @@ func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader
 	for i := range checkSums {
 		checkSums[i] = hex.EncodeToString(hashWriters[i].Sum(nil))
 	}
-	return bytesWritten, checkSums, nil
+	return newDisks, bytesWritten, checkSums, nil
 }
 
 // encodeData - encodes incoming data buffer into
@@ -108,7 +110,7 @@ func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, erro
 }
 
 // appendFile - append data buffer at path.
-func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hashWriters []hash.Hash, writeQuorum int) (err error) {
+func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hashWriters []hash.Hash, writeQuorum int) ([]StorageAPI, error) {
 	var wg = &sync.WaitGroup{}
 	var wErrs = make([]error, len(disks))
 	// Write encoded data to quorum disks in parallel.
@@ -124,8 +126,6 @@ func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hash
 			wErr := disk.AppendFile(volume, path, enBlocks[index])
 			if wErr != nil {
 				wErrs[index] = traceError(wErr)
-				// Ignore disk which returned an error.
-				disks[index] = nil
 				return
 			}
 
@@ -140,5 +140,5 @@ func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hash
 	// Wait for all the appends to finish.
 	wg.Wait()
 
-	return reduceWriteQuorumErrs(wErrs, objectOpIgnoredErrs, writeQuorum)
+	return evalDisks(disks, wErrs), reduceWriteQuorumErrs(wErrs, objectOpIgnoredErrs, writeQuorum)
 }

cmd/erasure-createfile_test.go

@@ -56,7 +56,7 @@ func TestErasureCreateFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Test when all disks are up.
-	size, _, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, _, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -69,7 +69,7 @@ func TestErasureCreateFile(t *testing.T) {
 	disks[5] = AppendDiskDown{disks[5].(*posix)}
 
 	// Test when two disks are down.
-	size, _, err = erasureCreateFile(disks, "testbucket", "testobject2", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, _, err = erasureCreateFile(disks, "testbucket", "testobject2", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -83,7 +83,7 @@ func TestErasureCreateFile(t *testing.T) {
 	disks[8] = AppendDiskDown{disks[8].(*posix)}
 	disks[9] = AppendDiskDown{disks[9].(*posix)}
 
-	size, _, err = erasureCreateFile(disks, "testbucket", "testobject3", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, _, err = erasureCreateFile(disks, "testbucket", "testobject3", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -93,7 +93,7 @@ func TestErasureCreateFile(t *testing.T) {
 
 	// 1 more disk down. 7 disk down in total. Should return quorum error.
 	disks[10] = AppendDiskDown{disks[10].(*posix)}
-	_, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, _, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if errorCause(err) != errXLWriteQuorum {
 		t.Errorf("erasureCreateFile return value: expected errXLWriteQuorum, got %s", err)
 	}

cmd/erasure-healfile.go

@@ -19,7 +19,9 @@ package cmd
 import "encoding/hex"
 
 // Heals the erasure coded file. reedsolomon.Reconstruct() is used to reconstruct the missing parts.
-func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string, size int64, blockSize int64, dataBlocks int, parityBlocks int, algo string) (checkSums []string, err error) {
+func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string,
+	size, blockSize int64, dataBlocks, parityBlocks int, algo HashAlgo) (checkSums []string, err error) {
+
 	var offset int64
 	remainingSize := size
 

cmd/erasure-healfile_test.go

@@ -48,7 +48,7 @@ func TestErasureHealFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Create a test file.
-	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/erasure-readfile.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"encoding/hex"
 	"errors"
 	"io"
 	"sync"
@@ -111,7 +110,9 @@ func getReadDisks(orderedDisks []StorageAPI, index int, dataBlocks int) (readDis
 }
 
 // parallelRead - reads chunks in parallel from the disks specified in []readDisks.
-func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []StorageAPI, enBlocks [][]byte, blockOffset int64, curChunkSize int64, bitRotVerify func(diskIndex int) bool, pool *bpool.BytePool) {
+func parallelRead(volume, path string, readDisks, orderedDisks []StorageAPI, enBlocks [][]byte,
+	blockOffset, curChunkSize int64, brVerifiers []bitRotVerifier, pool *bpool.BytePool) {
+
 	// WaitGroup to synchronise the read go-routines.
 	wg := &sync.WaitGroup{}
 
@@ -125,11 +126,15 @@ func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []St
 		go func(index int) {
 			defer wg.Done()
 
-			// Verify bit rot for the file on this disk.
-			if !bitRotVerify(index) {
-				// So that we don't read from this disk for the next block.
-				orderedDisks[index] = nil
-				return
+			// evaluate if we need to perform bit-rot checking
+			needBitRotVerification := true
+			if brVerifiers[index].isVerified {
+				needBitRotVerification = false
+				// if file has bit-rot, do not reuse disk
+				if brVerifiers[index].hasBitRot {
+					orderedDisks[index] = nil
+					return
+				}
 			}
 
 			buf, err := pool.Get()
@@ -140,7 +145,25 @@ func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []St
 			}
 			buf = buf[:curChunkSize]
 
-			_, err = readDisks[index].ReadFile(volume, path, blockOffset, buf)
+			if needBitRotVerification {
+				_, err = readDisks[index].ReadFileWithVerify(
+					volume, path, blockOffset, buf,
+					brVerifiers[index].algo,
+					brVerifiers[index].checkSum)
+			} else {
+				_, err = readDisks[index].ReadFile(volume, path,
+					blockOffset, buf)
+			}
+
+			// if bit-rot verification was done, store the
+			// result of verification so we can skip
+			// re-doing it next time
+			if needBitRotVerification {
+				brVerifiers[index].isVerified = true
+				_, ok := err.(hashMismatchError)
+				brVerifiers[index].hasBitRot = ok
+			}
+
 			if err != nil {
 				orderedDisks[index] = nil
 				return
@@ -153,12 +176,16 @@ func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []St
 	wg.Wait()
 }
 
-// erasureReadFile - read bytes from erasure coded files and writes to given writer.
-// Erasure coded files are read block by block as per given erasureInfo and data chunks
-// are decoded into a data block. Data block is trimmed for given offset and length,
-// then written to given writer. This function also supports bit-rot detection by
-// verifying checksum of individual block's checksum.
-func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path string, offset int64, length int64, totalLength int64, blockSize int64, dataBlocks int, parityBlocks int, checkSums []string, algo string, pool *bpool.BytePool) (int64, error) {
+// erasureReadFile - read bytes from erasure coded files and writes to
+// given writer.  Erasure coded files are read block by block as per
+// given erasureInfo and data chunks are decoded into a data
+// block. Data block is trimmed for given offset and length, then
+// written to given writer. This function also supports bit-rot
+// detection by verifying checksum of individual block's checksum.
+func erasureReadFile(writer io.Writer, disks []StorageAPI, volume, path string,
+	offset, length, totalLength, blockSize int64, dataBlocks, parityBlocks int,
+	checkSums []string, algo HashAlgo, pool *bpool.BytePool) (int64, error) {
+
 	// Offset and length cannot be negative.
 	if offset < 0 || length < 0 {
 		return 0, traceError(errUnexpected)
@@ -169,27 +196,15 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s
 		return 0, traceError(errUnexpected)
 	}
 
-	// chunkSize is the amount of data that needs to be read from each disk at a time.
+	// chunkSize is the amount of data that needs to be read from
+	// each disk at a time.
 	chunkSize := getChunkSize(blockSize, dataBlocks)
 
-	// bitRotVerify verifies if the file on a particular disk doesn't have bitrot
-	// by verifying the hash of the contents of the file.
-	bitRotVerify := func() func(diskIndex int) bool {
-		verified := make([]bool, len(disks))
-		// Return closure so that we have reference to []verified and
-		// not recalculate the hash on it every time the function is
-		// called for the same disk.
-		return func(diskIndex int) bool {
-			if verified[diskIndex] {
-				// Already validated.
-				return true
-			}
-			// Is this a valid block?
-			isValid := isValidBlock(disks[diskIndex], volume, path, checkSums[diskIndex], algo)
-			verified[diskIndex] = isValid
-			return isValid
-		}
-	}()
+	brVerifiers := make([]bitRotVerifier, len(disks))
+	for i := range brVerifiers {
+		brVerifiers[i].algo = algo
+		brVerifiers[i].checkSum = checkSums[i]
+	}
 
 	// Total bytes written to writer
 	var bytesWritten int64
@@ -241,7 +256,7 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s
 				return bytesWritten, err
 			}
 			// Issue a parallel read across the disks specified in readDisks.
-			parallelRead(volume, path, readDisks, disks, enBlocks, blockOffset, curChunkSize, bitRotVerify, pool)
+			parallelRead(volume, path, readDisks, disks, enBlocks, blockOffset, curChunkSize, brVerifiers, pool)
 			if isSuccessDecodeBlocks(enBlocks, dataBlocks) {
 				// If enough blocks are available to do rs.Reconstruct()
 				break
@@ -299,27 +314,6 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s
 	return bytesWritten, nil
 }
 
-// isValidBlock - calculates the checksum hash for the block and
-// validates if its correct returns true for valid cases, false otherwise.
-func isValidBlock(disk StorageAPI, volume, path, checkSum, checkSumAlgo string) (ok bool) {
-	// Disk is not available, not a valid block.
-	if disk == nil {
-		return false
-	}
-	// Checksum not available, not a valid block.
-	if checkSum == "" {
-		return false
-	}
-	// Read everything for a given block and calculate hash.
-	hashWriter := newHash(checkSumAlgo)
-	hashBytes, err := hashSum(disk, volume, path, hashWriter)
-	if err != nil {
-		errorIf(err, "Unable to calculate checksum %s/%s", volume, path)
-		return false
-	}
-	return hex.EncodeToString(hashBytes) == checkSum
-}
-
 // decodeData - decode encoded blocks.
 func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
 	// Initialized reedsolomon.

cmd/erasure-readfile_test.go

@@ -213,6 +213,12 @@ func (r ReadDiskDown) ReadFile(volume string, path string, offset int64, buf []b
 	return 0, errFaultyDisk
 }
 
+func (r ReadDiskDown) ReadFileWithVerify(volume string, path string, offset int64, buf []byte,
+	algo HashAlgo, expectedHash string) (n int64, err error) {
+
+	return 0, errFaultyDisk
+}
+
 func TestErasureReadFileDiskFail(t *testing.T) {
 	// Initialize environment needed for the test.
 	dataBlocks := 7
@@ -236,7 +242,7 @@ func TestErasureReadFileDiskFail(t *testing.T) {
 	}
 
 	// Create a test file to read from.
-	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -319,7 +325,7 @@ func TestErasureReadFileOffsetLength(t *testing.T) {
 	}
 
 	// Create a test file to read from.
-	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -398,7 +404,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	iterations := 10000
 
 	// Create a test file to read from.
-	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/erasure-utils.go

@@ -29,7 +29,7 @@ import (
 )
 
 // newHashWriters - inititialize a slice of hashes for the disk count.
-func newHashWriters(diskCount int, algo string) []hash.Hash {
+func newHashWriters(diskCount int, algo HashAlgo) []hash.Hash {
 	hashWriters := make([]hash.Hash, diskCount)
 	for index := range hashWriters {
 		hashWriters[index] = newHash(algo)
@@ -38,13 +38,13 @@ func newHashWriters(diskCount int, algo string) []hash.Hash {
 }
 
 // newHash - gives you a newly allocated hash depending on the input algorithm.
-func newHash(algo string) (h hash.Hash) {
+func newHash(algo HashAlgo) (h hash.Hash) {
 	switch algo {
-	case sha256Algo:
+	case HashSha256:
 		// sha256 checksum specially on ARM64 platforms or whenever
 		// requested as dictated by `xl.json` entry.
 		h = sha256.New()
-	case blake2bAlgo:
+	case HashBlake2b:
 		// ignore the error, because New512 without a key never fails
 		// New512 only returns a non-nil error, if the length of the passed
 		// key > 64 bytes - but we use blake2b as hash function (no key)
@@ -71,7 +71,7 @@ var hashBufferPool = sync.Pool{
 
 // hashSum calculates the hash of the entire path and returns.
 func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, error) {
-	// Fetch staging a new staging buffer from the pool.
+	// Fetch a new staging buffer from the pool.
 	bufp := hashBufferPool.Get().(*[]byte)
 	defer hashBufferPool.Put(bufp)
 
@@ -207,3 +207,16 @@ func copyBuffer(writer io.Writer, disk StorageAPI, volume string, path string, b
 	// Success.
 	return nil
 }
+
+// bitRotVerifier - type representing bit-rot verification process for
+// a single under-lying object (currently whole files)
+type bitRotVerifier struct {
+	// has the bit-rot verification been done?
+	isVerified bool
+	// is the data free of bit-rot?
+	hasBitRot bool
+	// hashing algorithm
+	algo HashAlgo
+	// hex-encoded expected raw-hash value
+	checkSum string
+}

cmd/event-notifier.go

@@ -172,7 +172,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
 	// For all other events we should set ETag and Size.
 	nEvent.S3.Object = objectMeta{
 		Key:         escapedObj,
-		ETag:        event.ObjInfo.MD5Sum,
+		ETag:        event.ObjInfo.ETag,
 		Size:        event.ObjInfo.Size,
 		ContentType: event.ObjInfo.ContentType,
 		UserDefined: event.ObjInfo.UserDefined,
@@ -503,9 +503,8 @@ func removeNotificationConfig(bucket string, objAPI ObjectLayer) error {
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
 	objLock.Lock()
-	err := objAPI.DeleteObject(minioMetaBucket, ncPath)
-	objLock.Unlock()
-	return err
+	defer objLock.Unlock()
+	return objAPI.DeleteObject(minioMetaBucket, ncPath)
 }
 
 // Remove listener configuration from storage layer. Used when a bucket is deleted.
@@ -516,9 +515,8 @@ func removeListenerConfig(bucket string, objAPI ObjectLayer) error {
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
 	objLock.Lock()
-	err := objAPI.DeleteObject(minioMetaBucket, lcPath)
-	objLock.Unlock()
-	return err
+	defer objLock.Unlock()
+	return objAPI.DeleteObject(minioMetaBucket, lcPath)
 }
 
 // Loads both notification and listener config.
@@ -609,6 +607,25 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 		}
 	}
 
+	// Load all mqtt targets, initialize their respective loggers.
+	for accountID, mqttN := range serverConfig.Notify.GetMQTT() {
+		if !mqttN.Enable {
+			continue
+		}
+
+		if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeMQTT, newMQTTNotify); err != nil {
+			if _, ok := err.(net.Error); ok {
+				err = &net.OpError{
+					Op:  "Connecting to " + queueARN,
+					Net: "tcp",
+					Err: err,
+				}
+			}
+
+			return nil, err
+		}
+	}
+
 	// Load all nats targets, initialize their respective loggers.
 	for accountID, natsN := range serverConfig.Notify.GetNATS() {
 		if !natsN.Enable {

cmd/event-notifier_test.go

@@ -45,7 +45,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
 	}
 
 	bucketName := "bucket"
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected error:", err)
 	}
 
@@ -312,7 +312,7 @@ func TestInitEventNotifier(t *testing.T) {
 	filterRules := []filterRule{
 		{
 			Name:  "prefix",
-			Value: globalMinioDefaultOwnerID,
+			Value: "minio",
 		},
 		{
 			Name:  "suffix",
@@ -343,7 +343,7 @@ func TestInitEventNotifier(t *testing.T) {
 	}
 
 	// create bucket
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected error:", err)
 	}
 
@@ -408,7 +408,7 @@ func TestListenBucketNotification(t *testing.T) {
 	objectName := "object"
 
 	// Create the bucket to listen on
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected error:", err)
 	}
 
@@ -518,7 +518,7 @@ func TestAddRemoveBucketListenerConfig(t *testing.T) {
 
 	// Make a bucket to store topicConfigs.
 	randBucket := getRandomBucketName()
-	if err := obj.MakeBucket(randBucket); err != nil {
+	if err := obj.MakeBucketWithLocation(randBucket, ""); err != nil {
 		t.Fatalf("Failed to make bucket %s", randBucket)
 	}
 
@@ -535,7 +535,7 @@ func TestAddRemoveBucketListenerConfig(t *testing.T) {
 	filterRules := []filterRule{
 		{
 			Name:  "prefix",
-			Value: globalMinioDefaultOwnerID,
+			Value: "minio",
 		},
 		{
 			Name:  "suffix",

cmd/format-config-v1.go

@@ -20,8 +20,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"reflect"
 	"sync"
+
+	"github.com/minio/minio/pkg/lock"
 )
 
 // fsFormat - structure holding 'fs' format.
@@ -29,6 +33,15 @@ type fsFormat struct {
 	Version string `json:"version"`
 }
 
+// FS format version strings.
+const (
+	// Represents the current backend disk structure
+	// version under `.minio.sys` and actual data namespace.
+
+	// formatConfigV1.fsFormat.Version
+	fsFormatBackendV1 = "1"
+)
+
 // xlFormat - structure holding 'xl' format.
 type xlFormat struct {
 	Version string `json:"version"` // Version of 'xl' format.
@@ -38,6 +51,15 @@ type xlFormat struct {
 	JBOD []string `json:"jbod"`
 }
 
+// XL format version strings.
+const (
+	// Represents the current backend disk structure
+	// version under `.minio.sys` and actual data namespace.
+
+	// formatConfigV1.xlFormat.Version
+	xlFormatBackendV1 = "1"
+)
+
 // formatConfigV1 - structure holds format config version '1'.
 type formatConfigV1 struct {
 	Version string `json:"version"` // Version of the format config.
@@ -47,6 +69,120 @@ type formatConfigV1 struct {
 	XL     *xlFormat `json:"xl,omitempty"` // XL field holds xl format.
 }
 
+// Format json file.
+const (
+	// Format config file carries backend format specific details.
+	formatConfigFile = "format.json"
+
+	// Format config tmp file carries backend format.
+	formatConfigFileTmp = "format.json.tmp"
+)
+
+// `format.json` version value.
+const (
+	// formatConfigV1.Version represents the version string
+	// of the current structure and its fields in `format.json`.
+	formatFileV1 = "1"
+
+	// Future `format.json` structure changes should have
+	// its own version and should be subsequently listed here.
+)
+
+// Constitutes `format.json` backend name.
+const (
+	// Represents FS backend.
+	formatBackendFS = "fs"
+
+	// Represents XL backend.
+	formatBackendXL = "xl"
+)
+
+// CheckFS if the format is FS and is valid with right values
+// returns appropriate errors otherwise.
+func (f *formatConfigV1) CheckFS() error {
+	// Validate if format config version is v1.
+	if f.Version != formatFileV1 {
+		return fmt.Errorf("Unknown format file version '%s'", f.Version)
+	}
+
+	// Validate if we have the expected format.
+	if f.Format != formatBackendFS {
+		return fmt.Errorf("FS backend format required. Found '%s'", f.Format)
+	}
+
+	// Check if format is currently supported.
+	if f.FS.Version != fsFormatBackendV1 {
+		return fmt.Errorf("Unknown backend FS format version '%s'", f.FS.Version)
+	}
+
+	// Success.
+	return nil
+}
+
+// LoadFormat - loads format config v1, returns `errUnformattedDisk`
+// if reading format.json fails with io.EOF.
+func (f *formatConfigV1) LoadFormat(lk *lock.LockedFile) error {
+	_, err := f.ReadFrom(lk)
+	if errorCause(err) == io.EOF {
+		// No data on disk `format.json` still empty
+		// treat it as unformatted disk.
+		return traceError(errUnformattedDisk)
+	}
+	return err
+}
+
+func (f *formatConfigV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
+	// Serialize to prepare to write to disk.
+	var fbytes []byte
+	fbytes, err = json.Marshal(f)
+	if err != nil {
+		return 0, traceError(err)
+	}
+	if err = lk.Truncate(0); err != nil {
+		return 0, traceError(err)
+	}
+	_, err = lk.Write(fbytes)
+	if err != nil {
+		return 0, traceError(err)
+	}
+	return int64(len(fbytes)), nil
+}
+
+func (f *formatConfigV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
+	var fbytes []byte
+	fi, err := lk.Stat()
+	if err != nil {
+		return 0, traceError(err)
+	}
+	fbytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
+	if err != nil {
+		return 0, traceError(err)
+	}
+	if len(fbytes) == 0 {
+		return 0, traceError(io.EOF)
+	}
+	// Decode `format.json`.
+	if err = json.Unmarshal(fbytes, f); err != nil {
+		return 0, traceError(err)
+	}
+	return int64(len(fbytes)), nil
+}
+
+func newFSFormat() (format *formatConfigV1) {
+	return newFSFormatV1()
+}
+
+// newFSFormatV1 - initializes new formatConfigV1 with FS format info.
+func newFSFormatV1() (format *formatConfigV1) {
+	return &formatConfigV1{
+		Version: formatFileV1,
+		Format:  formatBackendFS,
+		FS: &fsFormat{
+			Version: fsFormatBackendV1,
+		},
+	}
+}
+
 /*
 
 All disks online
@@ -770,10 +906,10 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
 
 func checkFormatXLValue(formatXL *formatConfigV1) error {
 	// Validate format version and format type.
-	if formatXL.Version != "1" {
+	if formatXL.Version != formatFileV1 {
 		return fmt.Errorf("Unsupported version of backend format [%s] found", formatXL.Version)
 	}
-	if formatXL.Format != "xl" {
+	if formatXL.Format != formatBackendXL {
 		return fmt.Errorf("Unsupported backend format [%s] found", formatXL.Format)
 	}
 	if formatXL.XL.Version != "1" {
@@ -875,10 +1011,10 @@ func initFormatXL(storageDisks []StorageAPI) (err error) {
 		}
 		// Allocate format config.
 		formats[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    mustGetUUID(),
 			},
 		}

cmd/format-config-v1_test.go

@@ -18,7 +18,12 @@ package cmd
 
 import (
 	"bytes"
+	"errors"
+	"os"
+	"path/filepath"
 	"testing"
+
+	"github.com/minio/minio/pkg/lock"
 )
 
 // generates a valid format.json for XL backend.
@@ -30,10 +35,10 @@ func genFormatXLValid() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -51,10 +56,10 @@ func genFormatXLInvalidVersion() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -75,10 +80,10 @@ func genFormatXLInvalidFormat() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -99,10 +104,10 @@ func genFormatXLInvalidXLVersion() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -116,8 +121,8 @@ func genFormatXLInvalidXLVersion() []*formatConfigV1 {
 
 func genFormatFS() *formatConfigV1 {
 	return &formatConfigV1{
-		Version: "1",
-		Format:  "fs",
+		Version: formatFileV1,
+		Format:  formatBackendFS,
 	}
 }
 
@@ -130,10 +135,10 @@ func genFormatXLInvalidJBODCount() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -151,10 +156,10 @@ func genFormatXLInvalidJBOD() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -178,10 +183,10 @@ func genFormatXLInvalidDisks() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -202,10 +207,10 @@ func genFormatXLInvalidDisksOrder() []*formatConfigV1 {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -223,7 +228,7 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
 	var err error
 	xl := obj.(*xlObjects)
 
-	err = obj.MakeBucket("bucket")
+	err = obj.MakeBucketWithLocation("bucket", "")
 	if err != nil {
 		return []StorageAPI{}, err
 	}
@@ -240,10 +245,10 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
 	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
 	// created when minio starts
 	for i := 3; i <= 5; i++ {
-		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 			return []StorageAPI{}, err
 		}
-		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "tmp"); err != nil {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, "tmp"); err != nil {
 			return []StorageAPI{}, err
 		}
 		if err = xl.storageDisks[i].DeleteFile(bucket, object+"/xl.json"); err != nil {
@@ -346,7 +351,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 
 	xl := obj.(*xlObjects)
 
-	err = obj.MakeBucket("bucket")
+	err = obj.MakeBucketWithLocation("bucket", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -361,19 +366,19 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 	}
 
 	// Now, remove two format files.. Load them and reorder
-	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
+	if err = xl.storageDisks[3].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 		t.Fatal(err)
 	}
-	if err = xl.storageDisks[11].DeleteFile(".minio.sys", "format.json"); err != nil {
+	if err = xl.storageDisks[11].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 		t.Fatal(err)
 	}
 
 	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
 	// created when minio starts
-	if err = xl.storageDisks[10].DeleteFile(".minio.sys", "format.json"); err != nil {
+	if err = xl.storageDisks[10].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 		t.Fatal(err)
 	}
-	if err = xl.storageDisks[10].DeleteFile(".minio.sys", "tmp"); err != nil {
+	if err = xl.storageDisks[10].DeleteFile(minioMetaBucket, "tmp"); err != nil {
 		t.Fatal(err)
 	}
 	if err = xl.storageDisks[10].DeleteFile(bucket, object+"/xl.json"); err != nil {
@@ -419,7 +424,7 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 
 	xl := obj.(*xlObjects)
 
-	err = obj.MakeBucket("bucket")
+	err = obj.MakeBucketWithLocation("bucket", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -434,10 +439,10 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 	}
 
 	// Now, remove two format files.. Load them and reorder
-	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
+	if err = xl.storageDisks[3].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 		t.Fatal(err)
 	}
-	if err = xl.storageDisks[5].DeleteFile(".minio.sys", "format.json"); err != nil {
+	if err = xl.storageDisks[5].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 		t.Fatal(err)
 	}
 
@@ -555,10 +560,10 @@ func TestSavedUUIDOrder(t *testing.T) {
 	}
 	for index := range jbod {
 		formatConfigs[index] = &formatConfigV1{
-			Version: "1",
-			Format:  "xl",
+			Version: formatFileV1,
+			Format:  formatBackendXL,
 			XL: &xlFormat{
-				Version: "1",
+				Version: xlFormatBackendV1,
 				Disk:    jbod[index],
 				JBOD:    jbod,
 			},
@@ -682,6 +687,163 @@ func TestGenericFormatCheckXL(t *testing.T) {
 	}
 }
 
+// TestFSCheckFormatFSErr - test loadFormatFS loading older format.
+func TestFSCheckFormatFSErr(t *testing.T) {
+	// Prepare for testing
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer removeAll(disk)
+
+	// Assign a new UUID.
+	uuid := mustGetUUID()
+
+	// Initialize meta volume, if volume already exists ignores it.
+	if err := initMetaVolumeFS(disk, uuid); err != nil {
+		t.Fatal(err)
+	}
+
+	testCases := []struct {
+		format         *formatConfigV1
+		formatWriteErr error
+		formatCheckErr error
+		shouldPass     bool
+	}{
+		{
+			format: &formatConfigV1{
+				Version: formatFileV1,
+				Format:  formatBackendFS,
+				FS: &fsFormat{
+					Version: fsFormatBackendV1,
+				},
+			},
+			formatCheckErr: nil,
+			shouldPass:     true,
+		},
+		{
+			format: &formatConfigV1{
+				Version: formatFileV1,
+				Format:  formatBackendFS,
+				FS: &fsFormat{
+					Version: "10",
+				},
+			},
+			formatCheckErr: errors.New("Unknown backend FS format version '10'"),
+			shouldPass:     false,
+		},
+		{
+			format: &formatConfigV1{
+				Version: formatFileV1,
+				Format:  "garbage",
+				FS: &fsFormat{
+					Version: fsFormatBackendV1,
+				},
+			},
+			formatCheckErr: errors.New("FS backend format required. Found 'garbage'"),
+		},
+		{
+			format: &formatConfigV1{
+				Version: "-1",
+				Format:  formatBackendFS,
+				FS: &fsFormat{
+					Version: fsFormatBackendV1,
+				},
+			},
+			formatCheckErr: errors.New("Unknown format file version '-1'"),
+		},
+	}
+
+	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
+	for i, testCase := range testCases {
+		lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = testCase.format.WriteTo(lk)
+		lk.Close()
+		if err != nil {
+			t.Fatalf("Test %d: Expected nil, got %s", i+1, err)
+		}
+
+		lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		formatCfg := &formatConfigV1{}
+		_, err = formatCfg.ReadFrom(lk)
+		lk.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = formatCfg.CheckFS()
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Should not fail with unexpected %s, expected nil", i+1, err)
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Should fail with expected %s, got nil", i+1, testCase.formatCheckErr)
+		}
+		if err != nil && !testCase.shouldPass {
+			if errorCause(err).Error() != testCase.formatCheckErr.Error() {
+				t.Errorf("Test %d: Should fail with expected %s, got %s", i+1, testCase.formatCheckErr, err)
+			}
+		}
+	}
+}
+
+// TestFSCheckFormatFS - test loadFormatFS with healty and faulty disks
+func TestFSCheckFormatFS(t *testing.T) {
+	// Prepare for testing
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer removeAll(disk)
+
+	// Assign a new UUID.
+	uuid := mustGetUUID()
+
+	// Initialize meta volume, if volume already exists ignores it.
+	if err := initMetaVolumeFS(disk, uuid); err != nil {
+		t.Fatal(err)
+	}
+
+	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
+	lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	format := newFSFormatV1()
+	_, err = format.WriteTo(lk)
+	lk.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Loading corrupted format file
+	file, err := os.OpenFile(preparePath(fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		t.Fatal("Should not fail here", err)
+	}
+	file.Write([]byte{'b'})
+	file.Close()
+
+	lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	format = &formatConfigV1{}
+	_, err = format.ReadFrom(lk)
+	lk.Close()
+	if err == nil {
+		t.Fatal("Should return an error here")
+	}
+
+	// Loading format file from disk not found.
+	removeAll(disk)
+	_, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDONLY, 0600)
+	if err != nil && !os.IsNotExist(err) {
+		t.Fatal("Should return 'format.json' does not exist, but got", err)
+	}
+}
+
 func TestLoadFormatXLErrs(t *testing.T) {
 	nDisks := 16
 	fsDirs, err := getRandomDisks(nDisks)
@@ -749,7 +911,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
 
 	// disks 0..10 returns unformatted disk
 	for i := 0; i <= 10; i++ {
-		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -873,7 +1035,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	}
 	xl = obj.(*xlObjects)
 	for i := 0; i <= 15; i++ {
-		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -894,7 +1056,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	}
 	xl = obj.(*xlObjects)
 	for i := 0; i <= 15; i++ {
-		if err = xl.storageDisks[i].AppendFile(".minio.sys", "format.json", []byte("corrupted data")); err != nil {
+		if err = xl.storageDisks[i].AppendFile(minioMetaBucket, formatConfigFile, []byte("corrupted data")); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -998,7 +1160,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	}
 	xl = obj.(*xlObjects)
 	for i := 0; i <= 15; i++ {
-		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
 			t.Fatal(err)
 		}
 	}

cmd/fs-v1-helpers.go

@@ -123,18 +123,27 @@ func fsMkdir(dirPath string) (err error) {
 	return nil
 }
 
-// Lookup if directory exists, returns directory
-// attributes upon success.
-func fsStatDir(statDir string) (os.FileInfo, error) {
-	if statDir == "" {
+func fsStat(statLoc string) (os.FileInfo, error) {
+	if statLoc == "" {
 		return nil, traceError(errInvalidArgument)
 	}
-	if err := checkPathLength(statDir); err != nil {
+	if err := checkPathLength(statLoc); err != nil {
+		return nil, traceError(err)
+	}
+	fi, err := osStat(preparePath(statLoc))
+	if err != nil {
 		return nil, traceError(err)
 	}
 
-	fi, err := osStat(preparePath(statDir))
+	return fi, nil
+}
+
+// Lookup if directory exists, returns directory
+// attributes upon success.
+func fsStatDir(statDir string) (os.FileInfo, error) {
+	fi, err := fsStat(statDir)
 	if err != nil {
+		err = errorCause(err)
 		if os.IsNotExist(err) {
 			return nil, traceError(errVolumeNotFound)
 		} else if os.IsPermission(err) {
@@ -152,16 +161,9 @@ func fsStatDir(statDir string) (os.FileInfo, error) {
 
 // Lookup if file exists, returns file attributes upon success
 func fsStatFile(statFile string) (os.FileInfo, error) {
-	if statFile == "" {
-		return nil, traceError(errInvalidArgument)
-	}
-
-	if err := checkPathLength(statFile); err != nil {
-		return nil, traceError(err)
-	}
-
-	fi, err := osStat(preparePath(statFile))
+	fi, err := fsStat(statFile)
 	if err != nil {
+		err = errorCause(err)
 		if os.IsNotExist(err) {
 			return nil, traceError(errFileNotFound)
 		} else if os.IsPermission(err) {
@@ -174,7 +176,7 @@ func fsStatFile(statFile string) (os.FileInfo, error) {
 		return nil, traceError(err)
 	}
 	if fi.IsDir() {
-		return nil, traceError(errFileNotFound)
+		return nil, traceError(errFileAccessDenied)
 	}
 	return fi, nil
 }
@@ -230,7 +232,7 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
 
 // Creates a file and copies data from incoming reader. Staging buffer is used by io.CopyBuffer.
 func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int64) (int64, error) {
-	if filePath == "" || reader == nil || buf == nil {
+	if filePath == "" || reader == nil {
 		return 0, traceError(errInvalidArgument)
 	}
 
@@ -263,11 +265,18 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
 		}
 	}
 
-	bytesWritten, err := io.CopyBuffer(writer, reader, buf)
-	if err != nil {
-		return 0, traceError(err)
+	var bytesWritten int64
+	if buf != nil {
+		bytesWritten, err = io.CopyBuffer(writer, reader, buf)
+		if err != nil {
+			return 0, traceError(err)
+		}
+	} else {
+		bytesWritten, err = io.Copy(writer, reader)
+		if err != nil {
+			return 0, traceError(err)
+		}
 	}
-
 	return bytesWritten, nil
 }
 
@@ -276,6 +285,12 @@ func fsRemoveUploadIDPath(basePath, uploadIDPath string) error {
 	if basePath == "" || uploadIDPath == "" {
 		return traceError(errInvalidArgument)
 	}
+	if err := checkPathLength(basePath); err != nil {
+		return traceError(err)
+	}
+	if err := checkPathLength(uploadIDPath); err != nil {
+		return traceError(err)
+	}
 
 	// List all the entries in uploadID.
 	entries, err := readDir(uploadIDPath)
@@ -319,6 +334,26 @@ func fsFAllocate(fd int, offset int64, len int64) (err error) {
 // Renames source path to destination path, creates all the
 // missing parents if they don't exist.
 func fsRenameFile(sourcePath, destPath string) error {
+	if err := checkPathLength(sourcePath); err != nil {
+		return traceError(err)
+	}
+	if err := checkPathLength(destPath); err != nil {
+		return traceError(err)
+	}
+	// Verify if source path exists.
+	if _, err := os.Stat(preparePath(sourcePath)); err != nil {
+		if os.IsNotExist(err) {
+			return traceError(errFileNotFound)
+		} else if os.IsPermission(err) {
+			return traceError(errFileAccessDenied)
+		} else if isSysErrPathNotFound(err) {
+			return traceError(errFileNotFound)
+		} else if isSysErrNotDir(err) {
+			// File path cannot be verified since one of the parents is a file.
+			return traceError(errFileAccessDenied)
+		}
+		return traceError(err)
+	}
 	if err := mkdirAll(pathutil.Dir(destPath), 0777); err != nil {
 		return traceError(err)
 	}

cmd/fs-v1-helpers_test.go

@@ -26,6 +26,31 @@ import (
 	"github.com/minio/minio/pkg/lock"
 )
 
+func TestFSRenameFile(t *testing.T) {
+	// create posix test setup
+	_, path, err := newPosixTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer removeAll(path)
+
+	if err = fsMkdir(pathJoin(path, "testvolume1")); err != nil {
+		t.Fatal(err)
+	}
+	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil {
+		t.Fatal(err)
+	}
+	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNotFound {
+		t.Fatal(err)
+	}
+	if err = fsRenameFile(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNameTooLong {
+		t.Fatal("Unexpected error", err)
+	}
+	if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
+		t.Fatal("Unexpected error", err)
+	}
+}
+
 func TestFSStats(t *testing.T) {
 	// create posix test setup
 	_, path, err := newPosixTestSetup()
@@ -48,9 +73,8 @@ func TestFSStats(t *testing.T) {
 		t.Fatalf("Unable to create volume, %s", err)
 	}
 
-	var buf = make([]byte, 4096)
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -60,7 +84,7 @@ func TestFSStats(t *testing.T) {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if _, err = fsCreateFile(pathJoin(path, "success-vol", "path/to/success-file"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(pathJoin(path, "success-vol", "path/to/success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -110,7 +134,7 @@ func TestFSStats(t *testing.T) {
 			srcFSPath:   path,
 			srcVol:      "success-vol",
 			srcPath:     "path",
-			expectedErr: errFileNotFound,
+			expectedErr: errFileAccessDenied,
 		},
 		// Test case - 6.
 		// Test case with src path segment > 255.
@@ -143,7 +167,8 @@ func TestFSStats(t *testing.T) {
 
 	for i, testCase := range testCases {
 		if testCase.srcPath != "" {
-			if _, err := fsStatFile(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
+			if _, err := fsStatFile(pathJoin(testCase.srcFSPath, testCase.srcVol,
+				testCase.srcPath)); errorCause(err) != testCase.expectedErr {
 				t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 			}
 		} else {
@@ -174,9 +199,8 @@ func TestFSCreateAndOpen(t *testing.T) {
 		t.Fatal("Unexpected error", err)
 	}
 
-	var buf = make([]byte, 4096)
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -204,7 +228,7 @@ func TestFSCreateAndOpen(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		_, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, buf, reader.Size())
+		_, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0)
 		if errorCause(err) != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
@@ -297,15 +321,14 @@ func TestFSRemoves(t *testing.T) {
 		t.Fatalf("Unable to create directory, %s", err)
 	}
 
-	var buf = make([]byte, 4096)
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
 	reader.Seek(0, 0)
 
-	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file-new"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file-new"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -417,9 +440,8 @@ func TestFSRemoveMeta(t *testing.T) {
 
 	filePath := pathJoin(fsPath, "success-vol", "success-file")
 
-	var buf = make([]byte, 4096)
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(filePath, reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(filePath, reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 

cmd/fs-v1-metadata.go

@@ -24,15 +24,31 @@ import (
 	pathutil "path"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/mimedb"
 	"github.com/tidwall/gjson"
 )
 
+// FS format, and object metadata.
 const (
-	fsMetaJSONFile   = "fs.json"
-	fsFormatJSONFile = "format.json"
+	// fs.json object metadata.
+	fsMetaJSONFile = "fs.json"
+)
+
+// FS metadata constants.
+const (
+	// FS backend meta 1.0.0 version.
+	fsMetaVersion100 = "1.0.0"
+
+	// FS backend meta 1.0.1 version.
+	fsMetaVersion = "1.0.1"
+
+	// FS backend meta format.
+	fsMetaFormat = "fs"
+
+	// Add more constants here.
 )
 
 // A fsMetaV1 represents a metadata header mapping keys to sets of values.
@@ -47,6 +63,19 @@ type fsMetaV1 struct {
 	Parts []objectPartInfo  `json:"parts,omitempty"`
 }
 
+// IsValid - tells if the format is sane by validating the version
+// string and format style.
+func (m fsMetaV1) IsValid() bool {
+	return isFSMetaValid(m.Version, m.Format)
+}
+
+// Verifies if the backend format metadata is sane by validating
+// the version string and format style.
+func isFSMetaValid(version, format string) bool {
+	return ((version == fsMetaVersion || version == fsMetaVersion100) &&
+		format == fsMetaFormat)
+}
+
 // Converts metadata to object info.
 func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo {
 	if len(m.Meta) == 0 {
@@ -75,17 +104,15 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 		objInfo.IsDir = fi.IsDir()
 	}
 
-	objInfo.MD5Sum = m.Meta["md5Sum"]
+	// Extract etag from metadata.
+	objInfo.ETag = extractETag(m.Meta)
 	objInfo.ContentType = m.Meta["content-type"]
 	objInfo.ContentEncoding = m.Meta["content-encoding"]
 
-	// md5Sum has already been extracted into objInfo.MD5Sum.  We
-	// need to remove it from m.Meta to avoid it from appearing as
-	// part of response headers. e.g, X-Minio-* or X-Amz-*.
-	delete(m.Meta, "md5Sum")
-
-	// Save all the other userdefined API.
-	objInfo.UserDefined = m.Meta
+	// etag/md5Sum has already been extracted. We need to
+	// remove to avoid it from appearing as part of
+	// response headers. e.g, X-Minio-* or X-Amz-*.
+	objInfo.UserDefined = cleanMetaETag(m.Meta)
 
 	// Success..
 	return objInfo
@@ -204,6 +231,12 @@ func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
 	// obtain format.
 	m.Format = parseFSFormat(fsMetaBuf)
 
+	// Verify if the format is valid, return corrupted format
+	// for unrecognized formats.
+	if !isFSMetaValid(m.Version, m.Format) {
+		return 0, traceError(errCorruptedFormat)
+	}
+
 	// obtain metadata.
 	m.Meta = parseFSMetaMap(fsMetaBuf)
 
@@ -217,17 +250,6 @@ func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
 	return int64(len(fsMetaBuf)), nil
 }
 
-// FS metadata constants.
-const (
-	// FS backend meta version.
-	fsMetaVersion = "1.0.0"
-
-	// FS backend meta format.
-	fsMetaFormat = "fs"
-
-	// Add more constants here.
-)
-
 // newFSMetaV1 - initializes new fsMetaV1.
 func newFSMetaV1() (fsMeta fsMetaV1) {
 	fsMeta = fsMetaV1{}
@@ -237,60 +259,97 @@ func newFSMetaV1() (fsMeta fsMetaV1) {
 	return fsMeta
 }
 
-// newFSFormatV1 - initializes new formatConfigV1 with FS format info.
-func newFSFormatV1() (format *formatConfigV1) {
-	return &formatConfigV1{
-		Version: "1",
-		Format:  "fs",
-		FS: &fsFormat{
-			Version: "1",
-		},
-	}
-}
+// Check if disk has already a valid format, holds a read lock and
+// upon success returns it to the caller to be closed.
+func checkLockedValidFormatFS(fsPath string) (*lock.RLockedFile, error) {
+	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
 
-// loads format.json from minioMetaBucket if it exists.
-func loadFormatFS(fsPath string) (*formatConfigV1, error) {
-	rlk, err := lock.RLockedOpenFile(pathJoin(fsPath, minioMetaBucket, fsFormatJSONFile))
+	rlk, err := lock.RLockedOpenFile(preparePath(fsFormatPath))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, errUnformattedDisk
+			// If format.json not found then
+			// its an unformatted disk.
+			return nil, traceError(errUnformattedDisk)
 		}
-		return nil, err
+		return nil, traceError(err)
 	}
-	defer rlk.Close()
 
-	formatBytes, err := ioutil.ReadAll(rlk)
-	if err != nil {
+	var format = &formatConfigV1{}
+	if err = format.LoadFormat(rlk.LockedFile); err != nil {
+		rlk.Close()
 		return nil, err
 	}
 
-	format := &formatConfigV1{}
-	if err = json.Unmarshal(formatBytes, format); err != nil {
+	// Check format FS.
+	if err = format.CheckFS(); err != nil {
+		rlk.Close()
 		return nil, err
 	}
 
-	return format, nil
+	//  Always return read lock here and should be closed by the caller.
+	return rlk, traceError(err)
 }
 
-// writes FS format (format.json) into minioMetaBucket.
-func saveFormatFS(formatPath string, fsFormat *formatConfigV1) error {
-	metadataBytes, err := json.Marshal(fsFormat)
-	if err != nil {
-		return err
-	}
+// Creates a new format.json if unformatted.
+func createFormatFS(fsPath string) error {
+	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
 
-	// fsFormatJSONFile - format.json file stored in minioMetaBucket(.minio.sys) directory.
-	lk, err := lock.LockedOpenFile(preparePath(formatPath), os.O_CREATE|os.O_WRONLY, 0600)
+	// Attempt a write lock on formatConfigFile `format.json`
+	// file stored in minioMetaBucket(.minio.sys) directory.
+	lk, err := lock.TryLockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
-		return err
+		return traceError(err)
 	}
+	// Close the locked file upon return.
 	defer lk.Close()
 
-	_, err = lk.Write(metadataBytes)
-	// Success.
+	// Load format on disk, checks if we are unformatted
+	// writes the new format.json
+	var format = &formatConfigV1{}
+	err = format.LoadFormat(lk)
+	if errorCause(err) == errUnformattedDisk {
+		_, err = newFSFormat().WriteTo(lk)
+		return err
+	}
 	return err
 }
 
+func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) {
+	// This loop validates format.json by holding a read lock and
+	// proceeds if disk unformatted to hold non-blocking WriteLock
+	// If for some reason non-blocking WriteLock fails and the error
+	// is lock.ErrAlreadyLocked i.e some other process is holding a
+	// lock we retry in the loop again.
+	for {
+		// Validate the `format.json` for expected values.
+		rlk, err = checkLockedValidFormatFS(fsPath)
+		switch {
+		case err == nil:
+			// Holding a read lock ensures that any write lock operation
+			// is blocked if attempted in-turn avoiding corruption on
+			// the backend disk.
+			return rlk, nil
+		case errorCause(err) == errUnformattedDisk:
+			if err = createFormatFS(fsPath); err != nil {
+				// Existing write locks detected.
+				if errorCause(err) == lock.ErrAlreadyLocked {
+					// Lock already present, sleep and attempt again.
+					time.Sleep(100 * time.Millisecond)
+					continue
+				}
+
+				// Unexpected error, return.
+				return nil, err
+			}
+
+			// Loop will continue to attempt a read-lock on `format.json`.
+		default:
+			// Unhandled error return.
+			return nil, err
+		}
+	}
+}
+
 // Return if the part info in uploadedParts and completeParts are same.
 func isPartsSame(uploadedParts []objectPartInfo, completeParts []completePart) bool {
 	if len(uploadedParts) != len(completeParts) {

cmd/fs-v1-metadata_test.go

@@ -48,7 +48,7 @@ func TestReadFSMetadata(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 	sha256sum := ""
@@ -58,7 +58,7 @@ func TestReadFSMetadata(t *testing.T) {
 	}
 
 	// Construct the full path of fs.json
-	fsPath := pathJoin("buckets", bucketName, objectName, "fs.json")
+	fsPath := pathJoin(bucketMetaPrefix, bucketName, objectName, "fs.json")
 	fsPath = pathJoin(fs.fsPath, minioMetaBucket, fsPath)
 
 	rlk, err := fs.rwPool.Open(fsPath)
@@ -85,7 +85,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 	sha256sum := ""
@@ -95,7 +95,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	}
 
 	// Construct the full path of fs.json
-	fsPath := pathJoin("buckets", bucketName, objectName, "fs.json")
+	fsPath := pathJoin(bucketMetaPrefix, bucketName, objectName, "fs.json")
 	fsPath = pathJoin(fs.fsPath, minioMetaBucket, fsPath)
 
 	rlk, err := fs.rwPool.Open(fsPath)
@@ -110,10 +110,10 @@ func TestWriteFSMetadata(t *testing.T) {
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
-	if fsMeta.Version != "1.0.0" {
+	if fsMeta.Version != fsMetaVersion {
 		t.Fatalf("Unexpected version %s", fsMeta.Version)
 	}
-	if fsMeta.Format != "fs" {
+	if fsMeta.Format != fsMetaFormat {
 		t.Fatalf("Unexpected format %s", fsMeta.Format)
 	}
 }

cmd/fs-v1-multipart.go

@@ -159,7 +159,7 @@ func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarke
 }
 
 // listMultipartUploads - lists all multipart uploads.
-func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
 	result := ListMultipartsInfo{}
 	recursive := true
 	if delimiter == slashSeparator {
@@ -191,7 +191,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 	if uploadIDMarker != "" {
 		uploads, _, err = fs.listMultipartUploadIDs(bucket, keyMarker, uploadIDMarker, maxUploads)
 		if err != nil {
-			return ListMultipartsInfo{}, err
+			return lmi, err
 		}
 		maxUploads = maxUploads - len(uploads)
 	}
@@ -232,7 +232,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 					eof = true
 					break
 				}
-				return ListMultipartsInfo{}, walkResult.err
+				return lmi, walkResult.err
 			}
 
 			entry := strings.TrimPrefix(walkResult.entry, retainSlash(bucket))
@@ -256,7 +256,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 
 			tmpUploads, end, err = fs.listMultipartUploadIDs(bucket, entry, uploadIDMarker, maxUploads)
 			if err != nil {
-				return ListMultipartsInfo{}, err
+				return lmi, err
 			}
 
 			uploads = append(uploads, tmpUploads...)
@@ -311,13 +311,13 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 // Implements S3 compatible ListMultipartUploads API. The resulting
 // ListMultipartsInfo structure is unmarshalled directly into XML and
 // replied back to the client.
-func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
 	if err := checkListMultipartArgs(bucket, prefix, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
-		return ListMultipartsInfo{}, err
+		return lmi, err
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return ListMultipartsInfo{}, toObjectErr(err, bucket)
+		return lmi, toObjectErr(err, bucket)
 	}
 
 	return fs.listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
@@ -412,9 +412,9 @@ func partToAppend(fsMeta fsMetaV1, fsAppendMeta fsMetaV1) (part objectPartInfo,
 // CopyObjectPart - similar to PutObjectPart but reads data from an existing
 // object. Internally incoming data is written to '.minio.sys/tmp' location
 // and safely renamed to '.minio.sys/multipart' for reach parts.
-func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64) (PartInfo, error) {
+func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64) (pi PartInfo, e error) {
 	if err := checkNewMultipartArgs(srcBucket, srcObject, fs); err != nil {
-		return PartInfo{}, err
+		return pi, err
 	}
 
 	// Initialize pipe.
@@ -431,7 +431,7 @@ func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, u
 
 	partInfo, err := fs.PutObjectPart(dstBucket, dstObject, uploadID, partID, length, pipeReader, "", "")
 	if err != nil {
-		return PartInfo{}, toObjectErr(err, dstBucket, dstObject)
+		return pi, toObjectErr(err, dstBucket, dstObject)
 	}
 
 	// Explicitly close the reader.
@@ -444,13 +444,13 @@ func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, u
 // an ongoing multipart transaction. Internally incoming data is
 // written to '.minio.sys/tmp' location and safely renamed to
 // '.minio.sys/multipart' for reach parts.
-func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (PartInfo, error) {
+func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (pi PartInfo, e error) {
 	if err := checkPutObjectPartArgs(bucket, object, fs); err != nil {
-		return PartInfo{}, err
+		return pi, err
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return PartInfo{}, toObjectErr(err, bucket)
+		return pi, toObjectErr(err, bucket)
 	}
 
 	// Hold the lock so that two parallel complete-multipart-uploads
@@ -463,9 +463,9 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	uploadsPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, object, uploadsJSONFile)
 	if _, err := fs.rwPool.Open(uploadsPath); err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return PartInfo{}, traceError(InvalidUploadID{UploadID: uploadID})
+			return pi, traceError(InvalidUploadID{UploadID: uploadID})
 		}
-		return PartInfo{}, toObjectErr(traceError(err), bucket, object)
+		return pi, toObjectErr(traceError(err), bucket, object)
 	}
 	defer fs.rwPool.Close(uploadsPath)
 
@@ -476,16 +476,16 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	rwlk, err := fs.rwPool.Write(fsMetaPath)
 	if err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return PartInfo{}, traceError(InvalidUploadID{UploadID: uploadID})
+			return pi, traceError(InvalidUploadID{UploadID: uploadID})
 		}
-		return PartInfo{}, toObjectErr(traceError(err), bucket, object)
+		return pi, toObjectErr(traceError(err), bucket, object)
 	}
 	defer rwlk.Close()
 
 	fsMeta := fsMetaV1{}
 	_, err = fsMeta.ReadFrom(rwlk)
 	if err != nil {
-		return PartInfo{}, toObjectErr(err, minioMetaMultipartBucket, fsMetaPath)
+		return pi, toObjectErr(err, minioMetaMultipartBucket, fsMetaPath)
 	}
 
 	partSuffix := fmt.Sprintf("object%d", partID)
@@ -523,14 +523,14 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	bytesWritten, cErr := fsCreateFile(fsPartPath, teeReader, buf, size)
 	if cErr != nil {
 		fsRemoveFile(fsPartPath)
-		return PartInfo{}, toObjectErr(cErr, minioMetaTmpBucket, tmpPartPath)
+		return pi, toObjectErr(cErr, minioMetaTmpBucket, tmpPartPath)
 	}
 
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.
 	if bytesWritten < size {
 		fsRemoveFile(fsPartPath)
-		return PartInfo{}, traceError(IncompleteBody{})
+		return pi, traceError(IncompleteBody{})
 	}
 
 	// Delete temporary part in case of failure. If
@@ -541,14 +541,14 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
 	if md5Hex != "" {
 		if newMD5Hex != md5Hex {
-			return PartInfo{}, traceError(BadDigest{md5Hex, newMD5Hex})
+			return pi, traceError(BadDigest{md5Hex, newMD5Hex})
 		}
 	}
 
 	if sha256sum != "" {
 		newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
 		if newSHA256sum != sha256sum {
-			return PartInfo{}, traceError(SHA256Mismatch{})
+			return pi, traceError(SHA256Mismatch{})
 		}
 	}
 
@@ -561,20 +561,20 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	fsNSPartPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, partPath)
 	if err = fsRenameFile(fsPartPath, fsNSPartPath); err != nil {
 		partLock.Unlock()
-		return PartInfo{}, toObjectErr(err, minioMetaMultipartBucket, partPath)
+		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
 	// Save the object part info in `fs.json`.
 	fsMeta.AddObjectPart(partID, partSuffix, newMD5Hex, size)
 	if _, err = fsMeta.WriteTo(rwlk); err != nil {
 		partLock.Unlock()
-		return PartInfo{}, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
+		return pi, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
 	}
 
 	partNamePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, partSuffix)
 	fi, err := fsStatFile(partNamePath)
 	if err != nil {
-		return PartInfo{}, toObjectErr(err, minioMetaMultipartBucket, partSuffix)
+		return pi, toObjectErr(err, minioMetaMultipartBucket, partSuffix)
 	}
 
 	// Append the part in background.
@@ -599,7 +599,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 // listObjectParts - wrapper scanning through
 // '.minio.sys/multipart/bucket/object/UPLOADID'. Lists all the parts
 // saved inside '.minio.sys/multipart/bucket/object/UPLOADID'.
-func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) {
+func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi ListPartsInfo, e error) {
 	result := ListPartsInfo{}
 
 	uploadIDPath := pathJoin(bucket, object, uploadID)
@@ -608,16 +608,16 @@ func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberM
 	if err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
 			// On windows oddly this is returned.
-			return ListPartsInfo{}, traceError(InvalidUploadID{UploadID: uploadID})
+			return lpi, traceError(InvalidUploadID{UploadID: uploadID})
 		}
-		return ListPartsInfo{}, toObjectErr(traceError(err), bucket, object)
+		return lpi, toObjectErr(traceError(err), bucket, object)
 	}
 	defer fs.rwPool.Close(fsMetaPath)
 
 	fsMeta := fsMetaV1{}
 	_, err = fsMeta.ReadFrom(metaFile.LockedFile)
 	if err != nil {
-		return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, fsMetaPath)
+		return lpi, toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
 
 	// Only parts with higher part numbers will be listed.
@@ -633,7 +633,7 @@ func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberM
 		partNamePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, part.Name)
 		fi, err = fsStatFile(partNamePath)
 		if err != nil {
-			return ListPartsInfo{}, toObjectErr(err, minioMetaMultipartBucket, partNamePath)
+			return lpi, toObjectErr(err, minioMetaMultipartBucket, partNamePath)
 		}
 		result.Parts = append(result.Parts, PartInfo{
 			PartNumber:   part.Number,
@@ -671,13 +671,13 @@ func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberM
 // Implements S3 compatible ListObjectParts API. The resulting
 // ListPartsInfo structure is unmarshalled directly into XML and
 // replied back to the client.
-func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) {
+func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi ListPartsInfo, e error) {
 	if err := checkListPartsArgs(bucket, object, fs); err != nil {
-		return ListPartsInfo{}, err
+		return lpi, err
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return ListPartsInfo{}, toObjectErr(err, bucket)
+		return lpi, toObjectErr(err, bucket)
 	}
 
 	// Hold the lock so that two parallel complete-multipart-uploads
@@ -688,7 +688,7 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
 
 	listPartsInfo, err := fs.listObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
 	if err != nil {
-		return ListPartsInfo{}, toObjectErr(err, bucket, object)
+		return lpi, toObjectErr(err, bucket, object)
 	}
 
 	// Success.
@@ -701,19 +701,24 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
 // md5sums of all the parts.
 //
 // Implements S3 compatible Complete multipart API.
-func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (ObjectInfo, error) {
+func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (oi ObjectInfo, e error) {
 	if err := checkCompleteMultipartArgs(bucket, object, fs); err != nil {
-		return ObjectInfo{}, err
+		return oi, err
+	}
+
+	// Check if an object is present as one of the parent dir.
+	if fs.parentDirIsObject(bucket, pathutil.Dir(object)) {
+		return oi, toObjectErr(traceError(errFileAccessDenied), bucket, object)
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket)
+		return oi, toObjectErr(err, bucket)
 	}
 
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5, err := getCompleteMultipartMD5(parts)
 	if err != nil {
-		return ObjectInfo{}, err
+		return oi, err
 	}
 
 	uploadIDPath := pathJoin(bucket, object, uploadID)
@@ -728,9 +733,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	rlk, err := fs.rwPool.Open(fsMetaPathMultipart)
 	if err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return ObjectInfo{}, traceError(InvalidUploadID{UploadID: uploadID})
+			return oi, traceError(InvalidUploadID{UploadID: uploadID})
 		}
-		return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(traceError(err), bucket, object)
 	}
 
 	// Disallow any parallel abort or complete multipart operations.
@@ -738,9 +743,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	if err != nil {
 		fs.rwPool.Close(fsMetaPathMultipart)
 		if err == errFileNotFound || err == errFileAccessDenied {
-			return ObjectInfo{}, traceError(InvalidUploadID{UploadID: uploadID})
+			return oi, traceError(InvalidUploadID{UploadID: uploadID})
 		}
-		return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(traceError(err), bucket, object)
 	}
 	defer rwlk.Close()
 
@@ -749,7 +754,31 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	_, err = fsMeta.ReadFrom(rlk.LockedFile)
 	if err != nil {
 		fs.rwPool.Close(fsMetaPathMultipart)
-		return ObjectInfo{}, toObjectErr(err, minioMetaMultipartBucket, fsMetaPathMultipart)
+		return oi, toObjectErr(err, minioMetaMultipartBucket, fsMetaPathMultipart)
+	}
+
+	// Validate all parts and then commit to disk.
+	for i, part := range parts {
+		partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
+		if partIdx == -1 {
+			fs.rwPool.Close(fsMetaPathMultipart)
+			return oi, traceError(InvalidPart{})
+		}
+
+		if fsMeta.Parts[partIdx].ETag != part.ETag {
+			fs.rwPool.Close(fsMetaPathMultipart)
+			return oi, traceError(InvalidPart{})
+		}
+
+		// All parts except the last part has to be atleast 5MB.
+		if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
+			fs.rwPool.Close(fsMetaPathMultipart)
+			return oi, traceError(PartTooSmall{
+				PartNumber: part.PartNumber,
+				PartSize:   fsMeta.Parts[partIdx].Size,
+				PartETag:   part.ETag,
+			})
+		}
 	}
 
 	// Wait for any competing PutObject() operation on bucket/object, since same namespace
@@ -758,7 +787,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	metaFile, err := fs.rwPool.Create(fsMetaPath)
 	if err != nil {
 		fs.rwPool.Close(fsMetaPathMultipart)
-		return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(traceError(err), bucket, object)
 	}
 	defer metaFile.Close()
 
@@ -775,7 +804,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 			fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID)
 			if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil {
 				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, toObjectErr(err, minioMetaTmpBucket, uploadID)
+				return oi, toObjectErr(err, minioMetaTmpBucket, uploadID)
 			}
 		}
 	}
@@ -793,29 +822,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		// Allocate staging buffer.
 		var buf = make([]byte, readSizeV1)
 
-		// Validate all parts and then commit to disk.
-		for i, part := range parts {
-			partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
-			if partIdx == -1 {
-				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, traceError(InvalidPart{})
-			}
-
-			if fsMeta.Parts[partIdx].ETag != part.ETag {
-				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, traceError(BadDigest{})
-			}
-
-			// All parts except the last part has to be atleast 5MB.
-			if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
-				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, traceError(PartTooSmall{
-					PartNumber: part.PartNumber,
-					PartSize:   fsMeta.Parts[partIdx].Size,
-					PartETag:   part.ETag,
-				})
-			}
-
+		for _, part := range parts {
 			// Construct part suffix.
 			partSuffix := fmt.Sprintf("object%d", part.PartNumber)
 			multipartPartFile := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, partSuffix)
@@ -826,9 +833,9 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 			if err != nil {
 				fs.rwPool.Close(fsMetaPathMultipart)
 				if err == errFileNotFound {
-					return ObjectInfo{}, traceError(InvalidPart{})
+					return oi, traceError(InvalidPart{})
 				}
-				return ObjectInfo{}, toObjectErr(traceError(err), minioMetaMultipartBucket, partSuffix)
+				return oi, toObjectErr(traceError(err), minioMetaMultipartBucket, partSuffix)
 			}
 
 			// No need to hold a lock, this is a unique file and will be only written
@@ -838,7 +845,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 			if err != nil {
 				reader.Close()
 				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+				return oi, toObjectErr(traceError(err), bucket, object)
 			}
 
 			_, err = io.CopyBuffer(wfile, reader, buf)
@@ -846,7 +853,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 				wfile.Close()
 				reader.Close()
 				fs.rwPool.Close(fsMetaPathMultipart)
-				return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+				return oi, toObjectErr(traceError(err), bucket, object)
 			}
 
 			wfile.Close()
@@ -855,7 +862,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 
 		if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil {
 			fs.rwPool.Close(fsMetaPathMultipart)
-			return ObjectInfo{}, toObjectErr(err, minioMetaTmpBucket, uploadID)
+			return oi, toObjectErr(err, minioMetaTmpBucket, uploadID)
 		}
 	}
 
@@ -866,12 +873,12 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	if len(fsMeta.Meta) == 0 {
 		fsMeta.Meta = make(map[string]string)
 	}
-	fsMeta.Meta["md5Sum"] = s3MD5
+	fsMeta.Meta["etag"] = s3MD5
 
 	// Write all the set metadata.
 	if _, err = fsMeta.WriteTo(metaFile); err != nil {
 		fs.rwPool.Close(fsMetaPathMultipart)
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
+		return oi, toObjectErr(err, bucket, object)
 	}
 
 	// Close lock held on bucket/object/uploadid/fs.json,
@@ -883,17 +890,17 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	multipartObjectDir := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, object)
 	multipartUploadIDDir := pathJoin(multipartObjectDir, uploadID)
 	if err = fsRemoveUploadIDPath(multipartObjectDir, multipartUploadIDDir); err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
+		return oi, toObjectErr(err, bucket, object)
 	}
 
 	// Remove entry from `uploads.json`.
 	if err = fs.removeUploadID(bucket, object, uploadID, rwlk); err != nil {
-		return ObjectInfo{}, toObjectErr(err, minioMetaMultipartBucket, pathutil.Join(bucket, object))
+		return oi, toObjectErr(err, minioMetaMultipartBucket, pathutil.Join(bucket, object))
 	}
 
 	fi, err := fsStatFile(fsNSObjPath)
 	if err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
+		return oi, toObjectErr(err, bucket, object)
 	}
 
 	// Return object info.

cmd/fs-v1-multipart_test.go

@@ -33,7 +33,7 @@ func TestFSWriteUploadJSON(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	obj.MakeBucket(bucketName)
+	obj.MakeBucketWithLocation(bucketName, "")
 	_, err := obj.NewMultipartUpload(bucketName, objectName, nil)
 	if err != nil {
 		t.Fatal("Unexpected err: ", err)
@@ -60,7 +60,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -91,7 +91,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	data := []byte("12345")
 	dataLen := int64(len(data))
 
-	if err = obj.MakeBucket(bucketName); err != nil {
+	if err = obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -122,7 +122,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -161,7 +161,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 

cmd/fs-v1.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"sort"
 	"syscall"
@@ -41,6 +42,9 @@ type fsObjects struct {
 	// temporary transactions.
 	fsUUID string
 
+	// This value shouldn't be touched, once initialized.
+	fsFormatRlk *lock.RLockedFile // Is a read lock on `format.json`.
+
 	// FS rw pool.
 	rwPool *fsIOPool
 
@@ -99,6 +103,16 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		}
 	}
 
+	di, err := getDiskInfo(preparePath(fsPath))
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if disk has minimum required total space.
+	if err = checkDiskMinTotal(di); err != nil {
+		return nil, err
+	}
+
 	// Assign a new UUID for FS minio mode. Each server instance
 	// gets its own UUID for temporary file transaction.
 	fsUUID := mustGetUUID()
@@ -108,24 +122,10 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		return nil, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
 	}
 
-	// Load `format.json`.
-	format, err := loadFormatFS(fsPath)
-	if err != nil && err != errUnformattedDisk {
-		return nil, fmt.Errorf("Unable to load 'format.json', %s", err)
-	}
-
-	// If the `format.json` doesn't exist create one.
-	if err == errUnformattedDisk {
-		fsFormatPath := pathJoin(fsPath, minioMetaBucket, fsFormatJSONFile)
-		// Initialize format.json, if already exists overwrite it.
-		if serr := saveFormatFS(fsFormatPath, newFSFormatV1()); serr != nil {
-			return nil, fmt.Errorf("Unable to initialize 'format.json', %s", serr)
-		}
-	}
-
-	// Validate if we have the same format.
-	if err == nil && format.Format != "fs" {
-		return nil, fmt.Errorf("Unable to recognize backend format, Disk is not in FS format. %s", format.Format)
+	// Initialize `format.json`, this function also returns.
+	rlk, err := initFormatFS(fsPath)
+	if err != nil {
+		return nil, err
 	}
 
 	// Initialize fs objects.
@@ -141,15 +141,19 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		},
 	}
 
+	// Once the filesystem has initialized hold the read lock for
+	// the life time of the server. This is done to ensure that under
+	// shared backend mode for FS, remote servers do not migrate
+	// or cause changes on backend format.
+	fs.fsFormatRlk = rlk
+
 	// Initialize and load bucket policies.
-	err = initBucketPolicies(fs)
-	if err != nil {
+	if err = initBucketPolicies(fs); err != nil {
 		return nil, fmt.Errorf("Unable to load all bucket policies. %s", err)
 	}
 
 	// Initialize a new event notifier.
-	err = initEventNotifier(fs)
-	if err != nil {
+	if err = initEventNotifier(fs); err != nil {
 		return nil, fmt.Errorf("Unable to initialize event notification. %s", err)
 	}
 
@@ -204,7 +208,7 @@ func (fs fsObjects) statBucketDir(bucket string) (os.FileInfo, error) {
 
 // MakeBucket - create a new bucket, returns if it
 // already exists.
-func (fs fsObjects) MakeBucket(bucket string) error {
+func (fs fsObjects) MakeBucketWithLocation(bucket, location string) error {
 	bucketDir, err := fs.getBucketDir(bucket)
 	if err != nil {
 		return toObjectErr(err, bucket)
@@ -218,10 +222,10 @@ func (fs fsObjects) MakeBucket(bucket string) error {
 }
 
 // GetBucketInfo - fetch bucket metadata info.
-func (fs fsObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
+func (fs fsObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
 	st, err := fs.statBucketDir(bucket)
 	if err != nil {
-		return BucketInfo{}, toObjectErr(err, bucket)
+		return bi, toObjectErr(err, bucket)
 	}
 
 	// As osStat() doesn't carry other than ModTime(), use ModTime() as CreatedTime.
@@ -308,15 +312,15 @@ func (fs fsObjects) DeleteBucket(bucket string) error {
 // CopyObject - copy object source object to destination object.
 // if source object and destination object are same we only
 // update metadata.
-func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string) (ObjectInfo, error) {
+func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string) (oi ObjectInfo, e error) {
 	if _, err := fs.statBucketDir(srcBucket); err != nil {
-		return ObjectInfo{}, toObjectErr(err, srcBucket)
+		return oi, toObjectErr(err, srcBucket)
 	}
 
 	// Stat the file to get file size.
 	fi, err := fsStatFile(pathJoin(fs.fsPath, srcBucket, srcObject))
 	if err != nil {
-		return ObjectInfo{}, toObjectErr(err, srcBucket, srcObject)
+		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
 
 	// Check if this request is only metadata update.
@@ -326,7 +330,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 		var wlk *lock.LockedFile
 		wlk, err = fs.rwPool.Write(fsMetaPath)
 		if err != nil {
-			return ObjectInfo{}, toObjectErr(traceError(err), srcBucket, srcObject)
+			return oi, toObjectErr(traceError(err), srcBucket, srcObject)
 		}
 		// This close will allow for locks to be synchronized on `fs.json`.
 		defer wlk.Close()
@@ -335,7 +339,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 		fsMeta := newFSMetaV1()
 		fsMeta.Meta = metadata
 		if _, err = fsMeta.WriteTo(wlk); err != nil {
-			return ObjectInfo{}, toObjectErr(err, srcBucket, srcObject)
+			return oi, toObjectErr(err, srcBucket, srcObject)
 		}
 
 		// Return the new object info.
@@ -360,7 +364,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 
 	objInfo, err := fs.PutObject(dstBucket, dstObject, length, pipeReader, metadata, "")
 	if err != nil {
-		return ObjectInfo{}, toObjectErr(err, dstBucket, dstObject)
+		return oi, toObjectErr(err, dstBucket, dstObject)
 	}
 
 	// Explicitly close the reader.
@@ -435,7 +439,7 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
 }
 
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
-func (fs fsObjects) getObjectInfo(bucket, object string) (ObjectInfo, error) {
+func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
 	fsMeta := fsMetaV1{}
 	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
 
@@ -450,44 +454,57 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (ObjectInfo, error) {
 			// PutObject() transaction, if we arrive at such
 			// a situation we just ignore and continue.
 			if errorCause(rerr) != io.EOF {
-				return ObjectInfo{}, toObjectErr(rerr, bucket, object)
+				return oi, toObjectErr(rerr, bucket, object)
 			}
 		}
 	}
 
 	// Ignore if `fs.json` is not available, this is true for pre-existing data.
 	if err != nil && err != errFileNotFound {
-		return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
+		return oi, toObjectErr(traceError(err), bucket, object)
 	}
 
 	// Stat the file to get file size.
 	fi, err := fsStatFile(pathJoin(fs.fsPath, bucket, object))
 	if err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
+		return oi, toObjectErr(err, bucket, object)
 	}
 
 	return fsMeta.ToObjectInfo(bucket, object, fi), nil
 }
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
-func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
-	// This is a special case with object whose name ends with
-	// a slash separator, we always return object not found here.
-	if hasSuffix(object, slashSeparator) {
-		return ObjectInfo{}, toObjectErr(traceError(errFileNotFound), bucket, object)
-	}
-
+func (fs fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
 	if err := checkGetObjArgs(bucket, object); err != nil {
-		return ObjectInfo{}, err
+		return oi, err
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket)
+		return oi, toObjectErr(err, bucket)
 	}
 
 	return fs.getObjectInfo(bucket, object)
 }
 
+// This function does the following check, suppose
+// object is "a/b/c/d", stat makes sure that objects ""a/b/c""
+// "a/b" and "a" do not exist.
+func (fs fsObjects) parentDirIsObject(bucket, parent string) bool {
+	var isParentDirObject func(string) bool
+	isParentDirObject = func(p string) bool {
+		if p == "." {
+			return false
+		}
+		if _, err := fsStatFile(pathJoin(fs.fsPath, bucket, p)); err == nil {
+			// If there is already a file at prefix "p" return error.
+			return true
+		}
+		// Check if there is a file as one of the parent paths.
+		return isParentDirObject(path.Dir(p))
+	}
+	return isParentDirObject(parent)
+}
+
 // PutObject - creates an object upon reading from the input stream
 // until EOF, writes data directly to configured filesystem path.
 // Additionally writes `fs.json` which carries the necessary metadata
@@ -495,18 +512,29 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
 func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, retErr error) {
 	var err error
 
-	// This is a special case with size as '0' and object ends with
-	// a slash separator, we treat it like a valid operation and
-	// return success.
+	// Validate if bucket name is valid and exists.
+	if _, err = fs.statBucketDir(bucket); err != nil {
+		return ObjectInfo{}, toObjectErr(err, bucket)
+	}
+
+	// This is a special case with size as '0' and object ends
+	// with a slash separator, we treat it like a valid operation
+	// and return success.
 	if isObjectDir(object, size) {
+		// Check if an object is present as one of the parent dir.
+		if fs.parentDirIsObject(bucket, path.Dir(object)) {
+			return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
+		}
 		return dirObjectInfo(bucket, object, size, metadata), nil
 	}
+
 	if err = checkPutObjectArgs(bucket, object, fs); err != nil {
 		return ObjectInfo{}, err
 	}
 
-	if _, err = fs.statBucketDir(bucket); err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket)
+	// Check if an object is present as one of the parent dir.
+	if fs.parentDirIsObject(bucket, path.Dir(object)) {
+		return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
 	}
 
 	// No metadata is set, allocate a new one.
@@ -592,12 +620,12 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 
 	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
 	// Update the md5sum if not set with the newly calculated one.
-	if len(metadata["md5Sum"]) == 0 {
-		metadata["md5Sum"] = newMD5Hex
+	if len(metadata["etag"]) == 0 {
+		metadata["etag"] = newMD5Hex
 	}
 
 	// md5Hex representation.
-	md5Hex := metadata["md5Sum"]
+	md5Hex := metadata["etag"]
 	if md5Hex != "" {
 		if newMD5Hex != md5Hex {
 			// Returns md5 mismatch.
@@ -729,24 +757,28 @@ func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
 		}
 	}
 
-	fsMetaMap := parseFSMetaMap(fsMetaBuf)
-	return fsMetaMap["md5Sum"], nil
+	// Check if FS metadata is valid, if not return error.
+	if !isFSMetaValid(parseFSVersion(fsMetaBuf), parseFSFormat(fsMetaBuf)) {
+		return "", toObjectErr(traceError(errCorruptedFormat), bucket, entry)
+	}
+
+	return extractETag(parseFSMetaMap(fsMetaBuf)), nil
 }
 
 // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
 // state for future re-entrant list requests.
-func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
 	if err := checkListObjsArgs(bucket, prefix, marker, delimiter, fs); err != nil {
-		return ListObjectsInfo{}, err
+		return loi, err
 	}
 
 	if _, err := fs.statBucketDir(bucket); err != nil {
-		return ListObjectsInfo{}, err
+		return loi, err
 	}
 
 	// With max keys of zero we have reached eof, return right here.
 	if maxKeys == 0 {
-		return ListObjectsInfo{}, nil
+		return loi, nil
 	}
 
 	// For delimiter and prefix as '/' we do not list anything at all
@@ -755,7 +787,7 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 	// as '/' we don't have any entries, since all the keys are
 	// of form 'keyName/...'
 	if delimiter == slashSeparator && prefix == slashSeparator {
-		return ListObjectsInfo{}, nil
+		return loi, nil
 	}
 
 	// Over flowing count - reset to maxObjectList.
@@ -781,8 +813,8 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 		// Protect reading `fs.json`.
 		objectLock := globalNSMutex.NewNSLock(bucket, entry)
 		objectLock.RLock()
-		var md5Sum string
-		md5Sum, err = fs.getObjectETag(bucket, entry)
+		var etag string
+		etag, err = fs.getObjectETag(bucket, entry)
 		objectLock.RUnlock()
 		if err != nil {
 			return ObjectInfo{}, err
@@ -802,7 +834,7 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 			Size:    fi.Size(),
 			ModTime: fi.ModTime(),
 			IsDir:   fi.IsDir(),
-			MD5Sum:  md5Sum,
+			ETag:    etag,
 		}, nil
 	}
 
@@ -836,13 +868,13 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 		if walkResult.err != nil {
 			// File not found is a valid case.
 			if errorCause(walkResult.err) == errFileNotFound {
-				return ListObjectsInfo{}, nil
+				return loi, nil
 			}
-			return ListObjectsInfo{}, toObjectErr(walkResult.err, bucket, prefix)
+			return loi, toObjectErr(walkResult.err, bucket, prefix)
 		}
 		objInfo, err := entryToObjectInfo(walkResult.entry)
 		if err != nil {
-			return ListObjectsInfo{}, nil
+			return loi, nil
 		}
 		nextMarker = objInfo.Name
 		objInfos = append(objInfos, objInfo)
@@ -884,8 +916,8 @@ func (fs fsObjects) HealBucket(bucket string) error {
 }
 
 // ListObjectsHeal - list all objects to be healed. Valid only for XL
-func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
-	return ListObjectsInfo{}, traceError(NotImplemented{})
+func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
+	return loi, traceError(NotImplemented{})
 }
 
 // ListBucketsHeal - list all buckets to be healed. Valid only for XL
@@ -894,6 +926,6 @@ func (fs fsObjects) ListBucketsHeal() ([]BucketInfo, error) {
 }
 
 func (fs fsObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
-	delimiter string, maxUploads int) (ListMultipartsInfo, error) {
-	return ListMultipartsInfo{}, traceError(NotImplemented{})
+	delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+	return lmi, traceError(NotImplemented{})
 }

cmd/fs-v1_test.go

@@ -63,7 +63,7 @@ func TestFSShutdown(t *testing.T) {
 		obj := initFSObjects(disk, t)
 		fs := obj.(*fsObjects)
 		objectContent := "12345"
-		obj.MakeBucket(bucketName)
+		obj.MakeBucketWithLocation(bucketName, "")
 		sha256sum := ""
 		obj.PutObject(bucketName, objectName, int64(len(objectContent)), bytes.NewReader([]byte(objectContent)), nil, sha256sum)
 		return fs, disk
@@ -85,47 +85,6 @@ func TestFSShutdown(t *testing.T) {
 	}
 }
 
-// TestFSLoadFormatFS - test loadFormatFS with healty and faulty disks
-func TestFSLoadFormatFS(t *testing.T) {
-	// Prepare for testing
-	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
-
-	// Assign a new UUID.
-	uuid := mustGetUUID()
-
-	// Initialize meta volume, if volume already exists ignores it.
-	if err := initMetaVolumeFS(disk, uuid); err != nil {
-		t.Fatal(err)
-	}
-
-	fsFormatPath := pathJoin(disk, minioMetaBucket, fsFormatJSONFile)
-	if err := saveFormatFS(preparePath(fsFormatPath), newFSFormatV1()); err != nil {
-		t.Fatal("Should not fail here", err)
-	}
-	_, err := loadFormatFS(disk)
-	if err != nil {
-		t.Fatal("Should not fail here", err)
-	}
-	// Loading corrupted format file
-	file, err := os.OpenFile(preparePath(fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
-	if err != nil {
-		t.Fatal("Should not fail here", err)
-	}
-	file.Write([]byte{'b'})
-	file.Close()
-	_, err = loadFormatFS(disk)
-	if err == nil {
-		t.Fatal("Should return an error here")
-	}
-	// Loading format file from disk not found.
-	removeAll(disk)
-	_, err = loadFormatFS(disk)
-	if err != nil && err != errUnformattedDisk {
-		t.Fatal("Should return unformatted disk, but got", err)
-	}
-}
-
 // TestFSGetBucketInfo - test GetBucketInfo with healty and faulty disks
 func TestFSGetBucketInfo(t *testing.T) {
 	// Prepare for testing
@@ -136,7 +95,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 	fs := obj.(*fsObjects)
 	bucketName := "bucket"
 
-	obj.MakeBucket(bucketName)
+	obj.MakeBucketWithLocation(bucketName, "")
 
 	// Test with valid parameters
 	info, err := fs.GetBucketInfo(bucketName)
@@ -161,6 +120,74 @@ func TestFSGetBucketInfo(t *testing.T) {
 	}
 }
 
+func TestFSPutObject(t *testing.T) {
+	// Prepare for tests
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer removeAll(disk)
+
+	obj := initFSObjects(disk, t)
+	bucketName := "bucket"
+	objectName := "1/2/3/4/object"
+
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
+		t.Fatal(err)
+	}
+
+	sha256sum := ""
+
+	// With a regular object.
+	_, err := obj.PutObject(bucketName+"non-existent", objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	if err == nil {
+		t.Fatal("Unexpected should fail here, bucket doesn't exist")
+	}
+	if _, ok := errorCause(err).(BucketNotFound); !ok {
+		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
+	}
+
+	// With a directory object.
+	_, err = obj.PutObject(bucketName+"non-existent", objectName+"/", int64(0), bytes.NewReader([]byte("")), nil, sha256sum)
+	if err == nil {
+		t.Fatal("Unexpected should fail here, bucket doesn't exist")
+	}
+	if _, ok := errorCause(err).(BucketNotFound); !ok {
+		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
+	}
+
+	_, err = obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = obj.PutObject(bucketName, objectName+"/1", int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	if err == nil {
+		t.Fatal("Unexpected should fail here, backend corruption occurred")
+	}
+	if nerr, ok := errorCause(err).(PrefixAccessDenied); !ok {
+		t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
+	} else {
+		if nerr.Bucket != "bucket" {
+			t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
+		}
+		if nerr.Object != "1/2/3/4/object/1" {
+			t.Fatalf("Expected '1/2/3/4/object/1', got %s", nerr.Object)
+		}
+	}
+
+	_, err = obj.PutObject(bucketName, objectName+"/1/", 0, bytes.NewReader([]byte("")), nil, sha256sum)
+	if err == nil {
+		t.Fatal("Unexpected should fail here, backned corruption occurred")
+	}
+	if nerr, ok := errorCause(err).(PrefixAccessDenied); !ok {
+		t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
+	} else {
+		if nerr.Bucket != "bucket" {
+			t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
+		}
+		if nerr.Object != "1/2/3/4/object/1/" {
+			t.Fatalf("Expected '1/2/3/4/object/1/', got %s", nerr.Object)
+		}
+	}
+}
+
 // TestFSDeleteObject - test fs.DeleteObject() with healthy and corrupted disks
 func TestFSDeleteObject(t *testing.T) {
 	// Prepare for tests
@@ -172,7 +199,7 @@ func TestFSDeleteObject(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	obj.MakeBucket(bucketName)
+	obj.MakeBucketWithLocation(bucketName, "")
 	sha256sum := ""
 	obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
 
@@ -217,7 +244,7 @@ func TestFSDeleteBucket(t *testing.T) {
 	fs := obj.(*fsObjects)
 	bucketName := "bucket"
 
-	err := obj.MakeBucket(bucketName)
+	err := obj.MakeBucketWithLocation(bucketName, "")
 	if err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
@@ -235,7 +262,7 @@ func TestFSDeleteBucket(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	obj.MakeBucket(bucketName)
+	obj.MakeBucketWithLocation(bucketName, "")
 
 	// Delete bucker should get error disk not found.
 	removeAll(disk)
@@ -256,7 +283,7 @@ func TestFSListBuckets(t *testing.T) {
 	fs := obj.(*fsObjects)
 
 	bucketName := "bucket"
-	if err := obj.MakeBucket(bucketName); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 

cmd/gateway-anonymous.go

@@ -0,0 +1,49 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "net/http"
+
+func anonErrToObjectErr(statusCode int, params ...string) error {
+	bucket := ""
+	object := ""
+	if len(params) >= 1 {
+		bucket = params[0]
+	}
+	if len(params) == 2 {
+		object = params[1]
+	}
+
+	switch statusCode {
+	case http.StatusNotFound:
+		if object != "" {
+			return ObjectNotFound{bucket, object}
+		}
+		return BucketNotFound{Bucket: bucket}
+	case http.StatusBadRequest:
+		if object != "" {
+			return ObjectNameInvalid{bucket, object}
+		}
+		return BucketNameInvalid{Bucket: bucket}
+	case http.StatusForbidden:
+		fallthrough
+	case http.StatusUnauthorized:
+		return AllAccessDisabled{bucket, object}
+	}
+
+	return errUnexpected
+}

cmd/gateway-azure-anonymous.go

@@ -29,14 +29,56 @@ import (
 	"github.com/Azure/azure-sdk-for-go/storage"
 )
 
+// Make anonymous HTTP request to azure endpoint.
+func azureAnonRequest(verb, urlStr string, header http.Header) (*http.Response, error) {
+	req, err := http.NewRequest(verb, urlStr, nil)
+	if err != nil {
+		return nil, err
+	}
+	if header != nil {
+		req.Header = header
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// 4XX and 5XX are error HTTP codes.
+	if resp.StatusCode >= 400 && resp.StatusCode <= 511 {
+		defer resp.Body.Close()
+		respBody, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(respBody) == 0 {
+			// no error in response body, might happen in HEAD requests
+			return nil, storage.AzureStorageServiceError{
+				StatusCode: resp.StatusCode,
+				Code:       resp.Status,
+				Message:    "no response body was available for error status code",
+			}
+		}
+		// Response contains Azure storage service error object.
+		var storageErr storage.AzureStorageServiceError
+		if err := xml.Unmarshal(respBody, &storageErr); err != nil {
+			return nil, err
+		}
+		storageErr.StatusCode = resp.StatusCode
+		return nil, storageErr
+	}
+
+	return resp, nil
+}
+
 // AnonGetBucketInfo - Get bucket metadata from azure anonymously.
-func (a AzureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
+func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
 	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
 	if err != nil {
 		return bucketInfo, azureToObjectError(traceError(err))
 	}
 	url.RawQuery = "restype=container"
-	resp, err := http.Head(url.String())
+	resp, err := azureAnonRequest(httpHEAD, url.String(), nil)
 	if err != nil {
 		return bucketInfo, azureToObjectError(traceError(err), bucket)
 	}
@@ -57,22 +99,24 @@ func (a AzureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, e
 	return bucketInfo, nil
 }
 
+// AnonPutObject - SendPUT request without authentication.
+// This is needed when clients send PUT requests on objects that can be uploaded without auth.
+func (a *azureObjects) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
+	// azure doesn't support anonymous put
+	return ObjectInfo{}, traceError(NotImplemented{})
+}
+
 // AnonGetObject - SendGET request without authentication.
 // This is needed when clients send GET requests on objects that can be downloaded without auth.
-func (a AzureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
-	u := a.client.GetBlobURL(bucket, object)
-	req, err := http.NewRequest("GET", u, nil)
-	if err != nil {
-		return azureToObjectError(traceError(err), bucket, object)
-	}
-
+func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
+	h := make(http.Header)
 	if length > 0 && startOffset > 0 {
-		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
+		h.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
 	} else if startOffset > 0 {
-		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
+		h.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := azureAnonRequest(httpGET, a.client.GetBlobURL(bucket, object), h)
 	if err != nil {
 		return azureToObjectError(traceError(err), bucket, object)
 	}
@@ -88,8 +132,8 @@ func (a AzureObjects) AnonGetObject(bucket, object string, startOffset int64, le
 
 // AnonGetObjectInfo - Send HEAD request without authentication and convert the
 // result to ObjectInfo.
-func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
-	resp, err := http.Head(a.client.GetBlobURL(bucket, object))
+func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
+	resp, err := azureAnonRequest(httpHEAD, a.client.GetBlobURL(bucket, object), nil)
 	if err != nil {
 		return objInfo, azureToObjectError(traceError(err), bucket, object)
 	}
@@ -120,7 +164,7 @@ func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectIn
 		objInfo.UserDefined["Content-Encoding"] = resp.Header.Get("Content-Encoding")
 	}
 	objInfo.UserDefined["Content-Type"] = resp.Header.Get("Content-Type")
-	objInfo.MD5Sum = resp.Header.Get("Etag")
+	objInfo.ETag = resp.Header.Get("Etag")
 	objInfo.ModTime = t
 	objInfo.Name = object
 	objInfo.Size = contentLength
@@ -128,7 +172,7 @@ func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectIn
 }
 
 // AnonListObjects - Use Azure equivalent ListBlobs.
-func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
+func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
 	params := storage.ListBlobsParameters{
 		Prefix:     prefix,
 		Marker:     marker,
@@ -146,6 +190,63 @@ func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 	}
 	url.RawQuery = q.Encode()
 
+	resp, err := azureAnonRequest(httpGET, url.String(), nil)
+	if err != nil {
+		return result, azureToObjectError(traceError(err))
+	}
+	defer resp.Body.Close()
+
+	var listResp storage.BlobListResponse
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return result, azureToObjectError(traceError(err))
+	}
+	err = xml.Unmarshal(data, &listResp)
+	if err != nil {
+		return result, azureToObjectError(traceError(err))
+	}
+
+	result.IsTruncated = listResp.NextMarker != ""
+	result.NextMarker = listResp.NextMarker
+	for _, object := range listResp.Blobs {
+		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
+		if e != nil {
+			continue
+		}
+		result.Objects = append(result.Objects, ObjectInfo{
+			Bucket:          bucket,
+			Name:            object.Name,
+			ModTime:         t,
+			Size:            object.Properties.ContentLength,
+			ETag:            object.Properties.Etag,
+			ContentType:     object.Properties.ContentType,
+			ContentEncoding: object.Properties.ContentEncoding,
+		})
+	}
+	result.Prefixes = listResp.BlobPrefixes
+	return result, nil
+}
+
+// AnonListObjectsV2 - List objects in V2 mode, anonymously
+func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) {
+	params := storage.ListBlobsParameters{
+		Prefix:     prefix,
+		Marker:     continuationToken,
+		Delimiter:  delimiter,
+		MaxResults: uint(maxKeys),
+	}
+
+	q := azureListBlobsGetParameters(params)
+	q.Set("restype", "container")
+	q.Set("comp", "list")
+
+	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
+	if err != nil {
+		return result, azureToObjectError(traceError(err))
+	}
+	url.RawQuery = q.Encode()
+
 	resp, err := http.Get(url.String())
 	if err != nil {
 		return result, azureToObjectError(traceError(err))
@@ -163,8 +264,11 @@ func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 		return result, azureToObjectError(traceError(err))
 	}
 
-	result.IsTruncated = listResp.NextMarker != ""
-	result.NextMarker = listResp.NextMarker
+	// If NextMarker is not empty, this means response is truncated and NextContinuationToken should be set
+	if listResp.NextMarker != "" {
+		result.IsTruncated = true
+		result.NextContinuationToken = listResp.NextMarker
+	}
 	for _, object := range listResp.Blobs {
 		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
 		if e != nil {
@@ -175,7 +279,7 @@ func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 			Name:            object.Name,
 			ModTime:         t,
 			Size:            object.Properties.ContentLength,
-			MD5Sum:          object.Properties.Etag,
+			ETag:            canonicalizeETag(object.Properties.Etag),
 			ContentType:     object.Properties.ContentType,
 			ContentEncoding: object.Properties.ContentEncoding,
 		})

cmd/gateway-azure-unsupported.go

@@ -17,27 +17,27 @@
 package cmd
 
 // HealBucket - Not relevant.
-func (a AzureObjects) HealBucket(bucket string) error {
+func (a *azureObjects) HealBucket(bucket string) error {
 	return traceError(NotImplemented{})
 }
 
 // ListBucketsHeal - Not relevant.
-func (a AzureObjects) ListBucketsHeal() (buckets []BucketInfo, err error) {
+func (a *azureObjects) ListBucketsHeal() (buckets []BucketInfo, err error) {
 	return nil, traceError(NotImplemented{})
 }
 
 // HealObject - Not relevant.
-func (a AzureObjects) HealObject(bucket, object string) (int, int, error) {
+func (a *azureObjects) HealObject(bucket, object string) (int, int, error) {
 	return 0, 0, traceError(NotImplemented{})
 }
 
 // ListObjectsHeal - Not relevant.
-func (a AzureObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
-	return ListObjectsInfo{}, traceError(NotImplemented{})
+func (a *azureObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
+	return loi, traceError(NotImplemented{})
 }
 
 // ListUploadsHeal - Not relevant.
-func (a AzureObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
-	delimiter string, maxUploads int) (ListMultipartsInfo, error) {
-	return ListMultipartsInfo{}, traceError(NotImplemented{})
+func (a *azureObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
+	delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+	return lmi, traceError(NotImplemented{})
 }

cmd/gateway-azure.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"crypto/md5"
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
@@ -36,6 +37,41 @@ import (
 
 const globalAzureAPIVersion = "2016-05-31"
 
+// Canonicalize the metadata headers, without this azure-sdk calculates
+// incorrect signature. This attempt to canonicalize is to convert
+// any HTTP header which is of form say `accept-encoding` should be
+// converted to `Accept-Encoding` in its canonical form.
+// Also replaces X-Amz-Meta prefix with X-Ms-Meta as Azure expects user
+// defined metadata to have X-Ms-Meta prefix.
+func s3ToAzureHeaders(headers map[string]string) (newHeaders map[string]string) {
+	newHeaders = make(map[string]string)
+	for k, v := range headers {
+		k = http.CanonicalHeaderKey(k)
+		if strings.HasPrefix(k, "X-Amz-Meta") {
+			k = strings.Replace(k, "X-Amz-Meta", "X-Ms-Meta", -1)
+		}
+		newHeaders[k] = v
+	}
+	return newHeaders
+}
+
+// Prefix user metadata with "X-Amz-Meta-".
+// client.GetBlobMetadata() already strips "X-Ms-Meta-"
+func azureToS3Metadata(meta map[string]string) (newMeta map[string]string) {
+	newMeta = make(map[string]string)
+
+	for k, v := range meta {
+		k = "X-Amz-Meta-" + k
+		newMeta[k] = v
+	}
+	return newMeta
+}
+
+// Append "-1" to etag so that clients do not interpret it as MD5.
+func azureToS3ETag(etag string) string {
+	return canonicalizeETag(etag) + "-1"
+}
+
 // To store metadata during NewMultipartUpload which will be used after
 // CompleteMultipartUpload to call SetBlobMetadata.
 type azureMultipartMetaInfo struct {
@@ -64,8 +100,8 @@ func (a *azureMultipartMetaInfo) del(key string) {
 	delete(a.meta, key)
 }
 
-// AzureObjects - Implements Object layer for Azure blob storage.
-type AzureObjects struct {
+// azureObjects - Implements Object layer for Azure blob storage.
+type azureObjects struct {
 	client   storage.BlobStorageClient // Azure sdk client
 	metaInfo azureMultipartMetaInfo
 }
@@ -106,6 +142,8 @@ func azureToObjectError(err error, params ...string) error {
 		err = BucketExists{Bucket: bucket}
 	case "InvalidResourceName":
 		err = BucketNameInvalid{Bucket: bucket}
+	case "RequestBodyTooLarge":
+		err = PartTooBig{}
 	default:
 		switch azureErr.StatusCode {
 		case http.StatusNotFound:
@@ -123,15 +161,26 @@ func azureToObjectError(err error, params ...string) error {
 }
 
 // Inits azure blob storage client and returns AzureObjects.
-func newAzureLayer(endPoint string, account, key string, secure bool) (GatewayLayer, error) {
-	if endPoint == "" {
-		endPoint = storage.DefaultBaseURL
+func newAzureLayer(host string) (GatewayLayer, error) {
+	var err error
+	var endpoint = storage.DefaultBaseURL
+	var secure = true
+
+	// If user provided some parameters
+	if host != "" {
+		endpoint, secure, err = parseGatewayEndpoint(host)
+		if err != nil {
+			return nil, err
+		}
 	}
-	c, err := storage.NewClient(account, key, endPoint, globalAzureAPIVersion, secure)
+
+	creds := serverConfig.GetCredential()
+	c, err := storage.NewClient(creds.AccessKey, creds.SecretKey, endpoint, globalAzureAPIVersion, secure)
 	if err != nil {
-		return AzureObjects{}, err
+		return &azureObjects{}, err
 	}
-	return &AzureObjects{
+
+	return &azureObjects{
 		client: c.GetBlobService(),
 		metaInfo: azureMultipartMetaInfo{
 			meta:  make(map[string]map[string]string),
@@ -142,36 +191,29 @@ func newAzureLayer(endPoint string, account, key string, secure bool) (GatewayLa
 
 // Shutdown - save any gateway metadata to disk
 // if necessary and reload upon next restart.
-func (a AzureObjects) Shutdown() error {
-	// TODO
+func (a *azureObjects) Shutdown() error {
 	return nil
 }
 
 // StorageInfo - Not relevant to Azure backend.
-func (a AzureObjects) StorageInfo() StorageInfo {
-	return StorageInfo{}
-}
-
-// MakeBucket - Create a new container on azure backend.
-func (a AzureObjects) MakeBucket(bucket string) error {
-	// will never be called, only satisfy ObjectLayer interface
-	return traceError(NotImplemented{})
+func (a *azureObjects) StorageInfo() (si StorageInfo) {
+	return si
 }
 
 // MakeBucketWithLocation - Create a new container on azure backend.
-func (a AzureObjects) MakeBucketWithLocation(bucket, location string) error {
+func (a *azureObjects) MakeBucketWithLocation(bucket, location string) error {
 	err := a.client.CreateContainer(bucket, storage.ContainerAccessTypePrivate)
 	return azureToObjectError(traceError(err), bucket)
 }
 
 // GetBucketInfo - Get bucket metadata..
-func (a AzureObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
+func (a *azureObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
 	// Azure does not have an equivalent call, hence use ListContainers.
 	resp, err := a.client.ListContainers(storage.ListContainersParameters{
 		Prefix: bucket,
 	})
 	if err != nil {
-		return BucketInfo{}, azureToObjectError(traceError(err), bucket)
+		return bi, azureToObjectError(traceError(err), bucket)
 	}
 	for _, container := range resp.Containers {
 		if container.Name == bucket {
@@ -184,11 +226,11 @@ func (a AzureObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
 			} // else continue
 		}
 	}
-	return BucketInfo{}, traceError(BucketNotFound{Bucket: bucket})
+	return bi, traceError(BucketNotFound{Bucket: bucket})
 }
 
 // ListBuckets - Lists all azure containers, uses Azure equivalent ListContainers.
-func (a AzureObjects) ListBuckets() (buckets []BucketInfo, err error) {
+func (a *azureObjects) ListBuckets() (buckets []BucketInfo, err error) {
 	resp, err := a.client.ListContainers(storage.ListContainersParameters{})
 	if err != nil {
 		return nil, azureToObjectError(traceError(err))
@@ -207,13 +249,13 @@ func (a AzureObjects) ListBuckets() (buckets []BucketInfo, err error) {
 }
 
 // DeleteBucket - delete a container on azure, uses Azure equivalent DeleteContainer.
-func (a AzureObjects) DeleteBucket(bucket string) error {
+func (a *azureObjects) DeleteBucket(bucket string) error {
 	return azureToObjectError(traceError(a.client.DeleteContainer(bucket)), bucket)
 }
 
 // ListObjects - lists all blobs on azure with in a container filtered by prefix
 // and marker, uses Azure equivalent ListBlobs.
-func (a AzureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
+func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
 	resp, err := a.client.ListBlobs(bucket, storage.ListBlobsParameters{
 		Prefix:     prefix,
 		Marker:     marker,
@@ -235,7 +277,42 @@ func (a AzureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxK
 			Name:            object.Name,
 			ModTime:         t,
 			Size:            object.Properties.ContentLength,
-			MD5Sum:          canonicalizeETag(object.Properties.Etag),
+			ETag:            azureToS3ETag(object.Properties.Etag),
+			ContentType:     object.Properties.ContentType,
+			ContentEncoding: object.Properties.ContentEncoding,
+		})
+	}
+	result.Prefixes = resp.BlobPrefixes
+	return result, nil
+}
+
+// ListObjectsV2 - list all blobs in Azure bucket filtered by prefix
+func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) {
+	resp, err := a.client.ListBlobs(bucket, storage.ListBlobsParameters{
+		Prefix:     prefix,
+		Marker:     continuationToken,
+		Delimiter:  delimiter,
+		MaxResults: uint(maxKeys),
+	})
+	if err != nil {
+		return result, azureToObjectError(traceError(err), bucket, prefix)
+	}
+	// If NextMarker is not empty, this means response is truncated and NextContinuationToken should be set
+	if resp.NextMarker != "" {
+		result.IsTruncated = true
+		result.NextContinuationToken = resp.NextMarker
+	}
+	for _, object := range resp.Blobs {
+		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
+		if e != nil {
+			continue
+		}
+		result.Objects = append(result.Objects, ObjectInfo{
+			Bucket:          bucket,
+			Name:            object.Name,
+			ModTime:         t,
+			Size:            object.Properties.ContentLength,
+			ETag:            azureToS3ETag(object.Properties.Etag),
 			ContentType:     object.Properties.ContentType,
 			ContentEncoding: object.Properties.ContentEncoding,
 		})
@@ -250,7 +327,7 @@ func (a AzureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxK
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (a AzureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
+func (a *azureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
 	byteRange := fmt.Sprintf("%d-", startOffset)
 	if length > 0 && startOffset > 0 {
 		byteRange = fmt.Sprintf("%d-%d", startOffset, startOffset+length-1)
@@ -273,7 +350,14 @@ func (a AzureObjects) GetObject(bucket, object string, startOffset int64, length
 
 // GetObjectInfo - reads blob metadata properties and replies back ObjectInfo,
 // uses zure equivalent GetBlobProperties.
-func (a AzureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
+func (a *azureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
+	blobMeta, err := a.client.GetBlobMetadata(bucket, object)
+	if err != nil {
+		return objInfo, azureToObjectError(traceError(err), bucket, object)
+	}
+
+	meta := azureToS3Metadata(blobMeta)
+
 	prop, err := a.client.GetBlobProperties(bucket, object)
 	if err != nil {
 		return objInfo, azureToObjectError(traceError(err), bucket, object)
@@ -282,55 +366,69 @@ func (a AzureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo,
 	if err != nil {
 		return objInfo, traceError(err)
 	}
+
+	if prop.ContentEncoding != "" {
+		meta["Content-Encoding"] = prop.ContentEncoding
+	}
+	meta["Content-Type"] = prop.ContentType
+
 	objInfo = ObjectInfo{
 		Bucket:      bucket,
-		UserDefined: make(map[string]string),
-		MD5Sum:      canonicalizeETag(prop.Etag),
+		UserDefined: meta,
+		ETag:        azureToS3ETag(prop.Etag),
 		ModTime:     t,
 		Name:        object,
 		Size:        prop.ContentLength,
 	}
-	if prop.ContentEncoding != "" {
-		objInfo.UserDefined["Content-Encoding"] = prop.ContentEncoding
-	}
-	objInfo.UserDefined["Content-Type"] = prop.ContentType
-	return objInfo, nil
-}
 
-// Canonicalize the metadata headers, without this azure-sdk calculates
-// incorrect signature. This attempt to canonicalize is to convert
-// any HTTP header which is of form say `accept-encoding` should be
-// converted to `Accept-Encoding` in its canonical form.
-func canonicalMetadata(metadata map[string]string) (canonical map[string]string) {
-	canonical = make(map[string]string)
-	for k, v := range metadata {
-		canonical[http.CanonicalHeaderKey(k)] = v
-	}
-	return canonical
+	return objInfo, nil
 }
 
 // PutObject - Create a new blob with the incoming data,
 // uses Azure equivalent CreateBlockBlobFromReader.
-func (a AzureObjects) PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
+func (a *azureObjects) PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
 	var sha256Writer hash.Hash
+	var md5sumWriter hash.Hash
+
+	var writers []io.Writer
+
+	md5sum := metadata["etag"]
+	delete(metadata, "etag")
+
 	teeReader := data
+
 	if sha256sum != "" {
 		sha256Writer = sha256.New()
-		teeReader = io.TeeReader(data, sha256Writer)
+		writers = append(writers, sha256Writer)
 	}
 
-	delete(metadata, "md5Sum")
+	if md5sum != "" {
+		md5sumWriter = md5.New()
+		writers = append(writers, md5sumWriter)
+	}
 
-	err = a.client.CreateBlockBlobFromReader(bucket, object, uint64(size), teeReader, canonicalMetadata(metadata))
+	if len(writers) > 0 {
+		teeReader = io.TeeReader(data, io.MultiWriter(writers...))
+	}
+
+	err = a.client.CreateBlockBlobFromReader(bucket, object, uint64(size), teeReader, s3ToAzureHeaders(metadata))
 	if err != nil {
 		return objInfo, azureToObjectError(traceError(err), bucket, object)
 	}
 
+	if md5sum != "" {
+		newMD5sum := hex.EncodeToString(md5sumWriter.Sum(nil))
+		if newMD5sum != md5sum {
+			a.client.DeleteBlob(bucket, object, nil)
+			return ObjectInfo{}, azureToObjectError(traceError(BadDigest{md5sum, newMD5sum}))
+		}
+	}
+
 	if sha256sum != "" {
 		newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
 		if newSHA256sum != sha256sum {
 			a.client.DeleteBlob(bucket, object, nil)
-			return ObjectInfo{}, traceError(SHA256Mismatch{})
+			return ObjectInfo{}, azureToObjectError(traceError(SHA256Mismatch{}))
 		}
 	}
 
@@ -339,7 +437,7 @@ func (a AzureObjects) PutObject(bucket, object string, size int64, data io.Reade
 
 // CopyObject - Copies a blob from source container to destination container.
 // Uses Azure equivalent CopyBlob API.
-func (a AzureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
+func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
 	err = a.client.CopyBlob(destBucket, destObject, a.client.GetBlobURL(srcBucket, srcObject))
 	if err != nil {
 		return objInfo, azureToObjectError(traceError(err), srcBucket, srcObject)
@@ -349,7 +447,7 @@ func (a AzureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject st
 
 // DeleteObject - Deletes a blob on azure container, uses Azure
 // equivalent DeleteBlob API.
-func (a AzureObjects) DeleteObject(bucket, object string) error {
+func (a *azureObjects) DeleteObject(bucket, object string) error {
 	err := a.client.DeleteBlob(bucket, object, nil)
 	if err != nil {
 		return azureToObjectError(traceError(err), bucket, object)
@@ -361,7 +459,7 @@ func (a AzureObjects) DeleteObject(bucket, object string) error {
 // FIXME: Full ListMultipartUploads is not supported yet. It is supported just enough to help our client libs to
 // support re-uploads. a.client.ListBlobs() can be made to return entries which include uncommitted blobs using
 // which we need to filter out the committed blobs to get the list of uncommitted blobs.
-func (a AzureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
+func (a *azureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
 	result.MaxUploads = maxUploads
 	result.Prefix = prefix
 	result.Delimiter = delimiter
@@ -377,7 +475,7 @@ func (a AzureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMa
 }
 
 // NewMultipartUpload - Use Azure equivalent CreateBlockBlob.
-func (a AzureObjects) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (a *azureObjects) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) {
 	// Azure doesn't return a unique upload ID and we use object name in place of it. Azure allows multiple uploads to
 	// co-exist as long as the user keeps the blocks uploaded (in block blobs) unique amongst concurrent upload attempts.
 	// Each concurrent client, keeps its own blockID list which it can commit.
@@ -386,14 +484,14 @@ func (a AzureObjects) NewMultipartUpload(bucket, object string, metadata map[str
 		// Store an empty map as a placeholder else ListObjectParts/PutObjectPart will not work properly.
 		metadata = make(map[string]string)
 	} else {
-		metadata = canonicalMetadata(metadata)
+		metadata = s3ToAzureHeaders(metadata)
 	}
 	a.metaInfo.set(uploadID, metadata)
 	return uploadID, nil
 }
 
 // CopyObjectPart - Not implemented.
-func (a AzureObjects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
+func (a *azureObjects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
 	return info, traceError(NotImplemented{})
 }
 
@@ -421,39 +519,66 @@ func azureParseBlockID(blockID string) (int, string, error) {
 }
 
 // PutObjectPart - Use Azure equivalent PutBlockWithLength.
-func (a AzureObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (info PartInfo, err error) {
+func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (info PartInfo, err error) {
 	if meta := a.metaInfo.get(uploadID); meta == nil {
 		return info, traceError(InvalidUploadID{})
 	}
 	var sha256Writer hash.Hash
+	var md5sumWriter hash.Hash
+	var etag string
+
+	var writers []io.Writer
+
 	if sha256sum != "" {
 		sha256Writer = sha256.New()
+		writers = append(writers, sha256Writer)
 	}
 
-	teeReader := io.TeeReader(data, sha256Writer)
+	if md5Hex != "" {
+		md5sumWriter = md5.New()
+		writers = append(writers, md5sumWriter)
+		etag = md5Hex
+	} else {
+		// Generate random ETag.
+		etag = getMD5Hash([]byte(mustGetUUID()))
+	}
 
-	id := azureGetBlockID(partID, md5Hex)
+	teeReader := data
+
+	if len(writers) > 0 {
+		teeReader = io.TeeReader(data, io.MultiWriter(writers...))
+	}
+
+	id := azureGetBlockID(partID, etag)
 	err = a.client.PutBlockWithLength(bucket, object, id, uint64(size), teeReader, nil)
 	if err != nil {
 		return info, azureToObjectError(traceError(err), bucket, object)
 	}
 
+	if md5Hex != "" {
+		newMD5sum := hex.EncodeToString(md5sumWriter.Sum(nil))
+		if newMD5sum != md5Hex {
+			a.client.DeleteBlob(bucket, object, nil)
+			return PartInfo{}, azureToObjectError(traceError(BadDigest{md5Hex, newMD5sum}))
+		}
+	}
+
 	if sha256sum != "" {
 		newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
 		if newSHA256sum != sha256sum {
-			return PartInfo{}, traceError(SHA256Mismatch{})
+			return PartInfo{}, azureToObjectError(traceError(SHA256Mismatch{}))
 		}
 	}
 
 	info.PartNumber = partID
-	info.ETag = md5Hex
+	info.ETag = etag
 	info.LastModified = UTCNow()
 	info.Size = size
 	return info, nil
 }
 
 // ListObjectParts - Use Azure equivalent GetBlockList.
-func (a AzureObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
+func (a *azureObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
 	result.Bucket = bucket
 	result.Object = object
 	result.UploadID = uploadID
@@ -502,13 +627,13 @@ func (a AzureObjects) ListObjectParts(bucket, object, uploadID string, partNumbe
 // AbortMultipartUpload - Not Implemented.
 // There is no corresponding API in azure to abort an incomplete upload. The uncommmitted blocks
 // gets deleted after one week.
-func (a AzureObjects) AbortMultipartUpload(bucket, object, uploadID string) error {
+func (a *azureObjects) AbortMultipartUpload(bucket, object, uploadID string) error {
 	a.metaInfo.del(uploadID)
 	return nil
 }
 
 // CompleteMultipartUpload - Use Azure equivalent PutBlockList.
-func (a AzureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (objInfo ObjectInfo, err error) {
+func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (objInfo ObjectInfo, err error) {
 	meta := a.metaInfo.get(uploadID)
 	if meta == nil {
 		return objInfo, traceError(InvalidUploadID{uploadID})
@@ -536,36 +661,15 @@ func (a AzureObjects) CompleteMultipartUpload(bucket, object, uploadID string, u
 		if err != nil {
 			return objInfo, azureToObjectError(traceError(err), bucket, object)
 		}
+		err = a.client.SetBlobMetadata(bucket, object, nil, meta)
+		if err != nil {
+			return objInfo, azureToObjectError(traceError(err), bucket, object)
+		}
 	}
 	a.metaInfo.del(uploadID)
 	return a.GetObjectInfo(bucket, object)
 }
 
-func anonErrToObjectErr(statusCode int, params ...string) error {
-	bucket := ""
-	object := ""
-	if len(params) >= 1 {
-		bucket = params[0]
-	}
-	if len(params) == 2 {
-		object = params[1]
-	}
-
-	switch statusCode {
-	case http.StatusNotFound:
-		if object != "" {
-			return ObjectNotFound{bucket, object}
-		}
-		return BucketNotFound{Bucket: bucket}
-	case http.StatusBadRequest:
-		if object != "" {
-			return ObjectNameInvalid{bucket, object}
-		}
-		return BucketNameInvalid{Bucket: bucket}
-	}
-	return errUnexpected
-}
-
 // Copied from github.com/Azure/azure-sdk-for-go/storage/blob.go
 func azureListBlobsGetParameters(p storage.ListBlobsParameters) url.Values {
 	out := url.Values{}
@@ -598,7 +702,7 @@ func azureListBlobsGetParameters(p storage.ListBlobsParameters) url.Values {
 // storage.ContainerAccessTypePrivate - none in minio terminology
 // As the common denominator for minio and azure is readonly and none, we support
 // these two policies at the bucket level.
-func (a AzureObjects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
+func (a *azureObjects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
 	var policies []BucketAccessPolicy
 
 	for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
@@ -626,7 +730,7 @@ func (a AzureObjects) SetBucketPolicies(bucket string, policyInfo policy.BucketA
 }
 
 // GetBucketPolicies - Get the container ACL and convert it to canonical []bucketAccessPolicy
-func (a AzureObjects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
+func (a *azureObjects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
 	policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
 	perm, err := a.client.GetContainerPermissions(bucket, 0, "")
 	if err != nil {
@@ -644,7 +748,7 @@ func (a AzureObjects) GetBucketPolicies(bucket string) (policy.BucketAccessPolic
 }
 
 // DeleteBucketPolicies - Set the container ACL to "private"
-func (a AzureObjects) DeleteBucketPolicies(bucket string) error {
+func (a *azureObjects) DeleteBucketPolicies(bucket string) error {
 	perm := storage.ContainerPermissions{
 		AccessType:     storage.ContainerAccessTypePrivate,
 		AccessPolicies: nil,

cmd/gateway-azure_test.go

@@ -18,25 +18,60 @@ package cmd
 
 import (
 	"net/http"
+	"net/url"
 	"reflect"
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
 )
 
+// Test azureToS3ETag.
+func TestAzureToS3ETag(t *testing.T) {
+	tests := []struct {
+		etag     string
+		expected string
+	}{
+		{`"etag"`, `etag-1`},
+		{"etag", "etag-1"},
+	}
+	for i, test := range tests {
+		got := azureToS3ETag(test.etag)
+		if got != test.expected {
+			t.Errorf("test %d: got:%s expected:%s", i+1, got, test.expected)
+		}
+	}
+}
+
 // Test canonical metadata.
-func TestCanonicalMetadata(t *testing.T) {
-	metadata := map[string]string{
+func TestS3ToAzureHeaders(t *testing.T) {
+	headers := map[string]string{
 		"accept-encoding":  "gzip",
 		"content-encoding": "gzip",
+		"X-Amz-Meta-Hdr":   "value",
 	}
-	expectedCanonicalM := map[string]string{
+	expectedHeaders := map[string]string{
 		"Accept-Encoding":  "gzip",
 		"Content-Encoding": "gzip",
+		"X-Ms-Meta-Hdr":    "value",
 	}
-	actualCanonicalM := canonicalMetadata(metadata)
-	if !reflect.DeepEqual(actualCanonicalM, expectedCanonicalM) {
-		t.Fatalf("Test failed, expected %#v, got %#v", expectedCanonicalM, actualCanonicalM)
+	actualHeaders := s3ToAzureHeaders(headers)
+	if !reflect.DeepEqual(actualHeaders, expectedHeaders) {
+		t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, actualHeaders)
+	}
+}
+
+func TestAzureToS3Metadata(t *testing.T) {
+	// Just one testcase. Adding more test cases does not add value to the testcase
+	// as azureToS3Metadata() just adds a prefix.
+	metadata := map[string]string{
+		"First-Name": "myname",
+	}
+	expectedMeta := map[string]string{
+		"X-Amz-Meta-First-Name": "myname",
+	}
+	actualMeta := azureToS3Metadata(metadata)
+	if !reflect.DeepEqual(actualMeta, expectedMeta) {
+		t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, actualMeta)
 	}
 }
 
@@ -140,3 +175,81 @@ func TestAzureParseBlockID(t *testing.T) {
 		t.Fatal("Expected azureParseBlockID() to return error")
 	}
 }
+
+// Test azureListBlobsGetParameters()
+func TestAzureListBlobsGetParameters(t *testing.T) {
+
+	// Test values set 1
+	expectedURLValues := url.Values{}
+	expectedURLValues.Set("prefix", "test")
+	expectedURLValues.Set("delimiter", "_")
+	expectedURLValues.Set("marker", "marker")
+	expectedURLValues.Set("include", "hello")
+	expectedURLValues.Set("maxresults", "20")
+	expectedURLValues.Set("timeout", "10")
+
+	setBlobParameters := storage.ListBlobsParameters{"test", "_", "marker", "hello", 20, 10}
+
+	// Test values set 2
+	expectedURLValues1 := url.Values{}
+
+	setBlobParameters1 := storage.ListBlobsParameters{"", "", "", "", 0, 0}
+
+	testCases := []struct {
+		name string
+		args storage.ListBlobsParameters
+		want url.Values
+	}{
+		{"TestIfValuesSet", setBlobParameters, expectedURLValues},
+		{"TestIfValuesNotSet", setBlobParameters1, expectedURLValues1},
+	}
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			if got := azureListBlobsGetParameters(test.args); !reflect.DeepEqual(got, test.want) {
+				t.Errorf("azureListBlobsGetParameters() = %v, want %v", got, test.want)
+			}
+		})
+	}
+}
+
+func TestAnonErrToObjectErr(t *testing.T) {
+	testCases := []struct {
+		name       string
+		statusCode int
+		params     []string
+		wantErr    error
+	}{
+		{"ObjectNotFound",
+			http.StatusNotFound,
+			[]string{"testBucket", "testObject"},
+			ObjectNotFound{Bucket: "testBucket", Object: "testObject"},
+		},
+		{"BucketNotFound",
+			http.StatusNotFound,
+			[]string{"testBucket", ""},
+			BucketNotFound{Bucket: "testBucket"},
+		},
+		{"ObjectNameInvalid",
+			http.StatusBadRequest,
+			[]string{"testBucket", "testObject"},
+			ObjectNameInvalid{Bucket: "testBucket", Object: "testObject"},
+		},
+		{"BucketNameInvalid",
+			http.StatusBadRequest,
+			[]string{"testBucket", ""},
+			BucketNameInvalid{Bucket: "testBucket"},
+		},
+		{"UnexpectedError",
+			http.StatusBadGateway,
+			[]string{"testBucket", "testObject"},
+			errUnexpected,
+		},
+	}
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			if err := anonErrToObjectErr(test.statusCode, test.params...); !reflect.DeepEqual(err, test.wantErr) {
+				t.Errorf("anonErrToObjectErr() error = %v, wantErr %v", err, test.wantErr)
+			}
+		})
+	}
+}

cmd/gateway-gcs-anonymous.go

@@ -0,0 +1,142 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+func toGCSPublicURL(bucket, object string) string {
+	return fmt.Sprintf("https://storage.googleapis.com/%s/%s", bucket, object)
+}
+
+// AnonPutObject creates a new object anonymously with the incoming data,
+func (l *gcsGateway) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error) {
+
+	return ObjectInfo{}, NotImplemented{}
+}
+
+// AnonGetObject - Get object anonymously
+func (l *gcsGateway) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
+	req, err := http.NewRequest("GET", toGCSPublicURL(bucket, object), nil)
+	if err != nil {
+		return gcsToObjectError(traceError(err), bucket, object)
+	}
+
+	if length > 0 && startOffset > 0 {
+		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
+	} else if startOffset > 0 {
+		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return gcsToObjectError(traceError(err), bucket, object)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
+		return gcsToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
+	}
+
+	_, err = io.Copy(writer, resp.Body)
+	return gcsToObjectError(traceError(err), bucket, object)
+}
+
+// AnonGetObjectInfo - Get object info anonymously
+func (l *gcsGateway) AnonGetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
+	resp, err := http.Head(toGCSPublicURL(bucket, object))
+	if err != nil {
+		return objInfo, gcsToObjectError(traceError(err), bucket, object)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return objInfo, gcsToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
+	}
+
+	var contentLength int64
+	contentLengthStr := resp.Header.Get("Content-Length")
+	if contentLengthStr != "" {
+		contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
+		if err != nil {
+			return objInfo, gcsToObjectError(traceError(errUnexpected), bucket, object)
+		}
+	}
+
+	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
+	if err != nil {
+		return objInfo, traceError(err)
+	}
+
+	objInfo.ModTime = t
+	objInfo.Bucket = bucket
+	objInfo.UserDefined = make(map[string]string)
+	if resp.Header.Get("Content-Encoding") != "" {
+		objInfo.UserDefined["Content-Encoding"] = resp.Header.Get("Content-Encoding")
+	}
+	objInfo.UserDefined["Content-Type"] = resp.Header.Get("Content-Type")
+	objInfo.ETag = resp.Header.Get("Etag")
+	objInfo.ModTime = t
+	objInfo.Name = object
+	objInfo.Size = contentLength
+	return
+}
+
+// AnonListObjects - List objects anonymously
+func (l *gcsGateway) AnonListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+	result, err := l.anonClient.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
+	if err != nil {
+		return ListObjectsInfo{}, s3ToObjectError(traceError(err), bucket)
+	}
+
+	return fromMinioClientListBucketResult(bucket, result), nil
+}
+
+// AnonListObjectsV2 - List objects in V2 mode, anonymously
+func (l *gcsGateway) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (ListObjectsV2Info, error) {
+	// Request V1 List Object to the backend
+	result, err := l.anonClient.ListObjects(bucket, prefix, continuationToken, delimiter, maxKeys)
+	if err != nil {
+		return ListObjectsV2Info{}, s3ToObjectError(traceError(err), bucket)
+	}
+	// translate V1 Result to V2Info
+	return fromMinioClientListBucketResultToV2Info(bucket, result), nil
+}
+
+// AnonGetBucketInfo - Get bucket metadata anonymously.
+func (l *gcsGateway) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
+	resp, err := http.Head(toGCSPublicURL(bucket, ""))
+	if err != nil {
+		return bucketInfo, gcsToObjectError(traceError(err))
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return bucketInfo, gcsToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket)), bucket)
+	}
+
+	// Last-Modified date being returned by GCS
+	return BucketInfo{
+		Name: bucket,
+	}, nil
+}

cmd/gateway-gcs-errors.go

@@ -0,0 +1,27 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "errors"
+
+var (
+	// Project ID format is not valid.
+	errGCSInvalidProjectID = errors.New("GCS project id is either empty or invalid")
+
+	// Project ID not found
+	errGCSProjectIDNotFound = errors.New("unknown project id")
+)

cmd/gateway-gcs-unsupported.go

@@ -0,0 +1,42 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// HealBucket - Not relevant.
+func (l *gcsGateway) HealBucket(bucket string) error {
+	return traceError(NotImplemented{})
+}
+
+// ListBucketsHeal - Not relevant.
+func (l *gcsGateway) ListBucketsHeal() (buckets []BucketInfo, err error) {
+	return []BucketInfo{}, traceError(NotImplemented{})
+}
+
+// HealObject - Not relevant.
+func (l *gcsGateway) HealObject(bucket string, object string) (int, int, error) {
+	return 0, 0, traceError(NotImplemented{})
+}
+
+// ListObjectsHeal - Not relevant.
+func (l *gcsGateway) ListObjectsHeal(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+	return ListObjectsInfo{}, traceError(NotImplemented{})
+}
+
+// ListUploadsHeal - Not relevant.
+func (l *gcsGateway) ListUploadsHeal(bucket string, prefix string, marker string, uploadIDMarker string, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+	return ListMultipartsInfo{}, traceError(NotImplemented{})
+}

cmd/gateway-gcs_test.go

@@ -0,0 +1,181 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	minio "github.com/minio/minio-go"
+)
+
+func TestToGCSPageToken(t *testing.T) {
+	testCases := []struct {
+		Name  string
+		Token string
+	}{
+		{
+			Name:  "A",
+			Token: "CgFB",
+		},
+		{
+			Name:  "AAAAAAAAAA",
+			Token: "CgpBQUFBQUFBQUFB",
+		},
+		{
+			Name:  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+			Token: "CmRBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB",
+		},
+		{
+			Name:  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+			Token: "CpEDQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUE=",
+		},
+		{
+			Name:  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+			Token: "CpIDQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB",
+		},
+		{
+			Name:  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+			Token: "CpMDQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQQ==",
+		},
+		{
+			Name:  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+			Token: "CvQDQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUE=",
+		},
+	}
+
+	for i, testCase := range testCases {
+		if toGCSPageToken(testCase.Name) != testCase.Token {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, toGCSPageToken(testCase.Name), testCase.Token)
+		}
+	}
+
+}
+
+// TestIsValidGCSProjectIDFormat tests isValidGCSProjectIDFormat
+func TestValidGCSProjectIDFormat(t *testing.T) {
+	testCases := []struct {
+		ProjectID string
+		Valid     bool
+	}{
+		{"", false},
+		{"a", false},
+		{"Abc", false},
+		{"1bcd", false},
+		// 5 chars
+		{"abcdb", false},
+		// 6 chars
+		{"abcdbz", true},
+		// 30 chars
+		{"project-id-1-project-id-more-1", true},
+		// 31 chars
+		{"project-id-1-project-id-more-11", false},
+		{"storage.googleapis.com", false},
+		{"http://storage.googleapis.com", false},
+		{"http://localhost:9000", false},
+		{"project-id-1", true},
+		{"project-id-1988832", true},
+		{"projectid1414", true},
+	}
+
+	for i, testCase := range testCases {
+		valid := isValidGCSProjectIDFormat(testCase.ProjectID)
+		if valid != testCase.Valid {
+			t.Errorf("Test %d: Expected %v, got %v", i+1, valid, testCase.Valid)
+		}
+	}
+}
+
+// Test for isGCSMarker.
+func TestIsGCSMarker(t *testing.T) {
+	testCases := []struct {
+		marker   string
+		expected bool
+	}{
+		{
+			marker:   "{minio}gcs123",
+			expected: true,
+		},
+		{
+			marker:   "{mini_no}tgcs123",
+			expected: false,
+		},
+		{
+			marker:   "{minioagainnotgcs123",
+			expected: false,
+		},
+		{
+			marker:   "obj1",
+			expected: false,
+		},
+	}
+
+	for i, tc := range testCases {
+		if actual := isGCSMarker(tc.marker); actual != tc.expected {
+			t.Errorf("Test %d: marker is %s, expected %v but got %v",
+				i+1, tc.marker, tc.expected, actual)
+		}
+	}
+}
+
+// Test for gcsMultipartMetaName.
+func TestGCSMultipartMetaName(t *testing.T) {
+	uploadID := "a"
+	expected := pathJoin(gcsMinioMultipartPathV1, uploadID, gcsMinioMultipartMeta)
+	got := gcsMultipartMetaName(uploadID)
+	if expected != got {
+		t.Errorf("expected: %s, got: %s", expected, got)
+	}
+}
+
+// Test for gcsMultipartDataName.
+func TestGCSMultipartDataName(t *testing.T) {
+	var (
+		uploadID   = "a"
+		etag       = "b"
+		partNumber = 1
+	)
+	expected := pathJoin(gcsMinioMultipartPathV1, uploadID, fmt.Sprintf("%05d.%s", partNumber, etag))
+	got := gcsMultipartDataName(uploadID, partNumber, etag)
+	if expected != got {
+		t.Errorf("expected: %s, got: %s", expected, got)
+	}
+}
+
+func TestFromMinioClientListBucketResultToV2Info(t *testing.T) {
+
+	listBucketResult := minio.ListBucketResult{
+		IsTruncated:    false,
+		Marker:         "testMarker",
+		NextMarker:     "testMarker2",
+		CommonPrefixes: []minio.CommonPrefix{{Prefix: "one"}, {Prefix: "two"}},
+		Contents:       []minio.ObjectInfo{{Key: "testobj", ContentType: ""}},
+	}
+
+	listBucketV2Info := ListObjectsV2Info{
+		Prefixes:              []string{"one", "two"},
+		Objects:               []ObjectInfo{{Name: "testobj", Bucket: "testbucket", UserDefined: map[string]string{"Content-Type": ""}}},
+		IsTruncated:           false,
+		ContinuationToken:     "testMarker",
+		NextContinuationToken: "testMarker2",
+	}
+
+	if got := fromMinioClientListBucketResultToV2Info("testbucket", listBucketResult); !reflect.DeepEqual(got, listBucketV2Info) {
+		t.Errorf("fromMinioClientListBucketResultToV2Info() = %v, want %v", got, listBucketV2Info)
+	}
+}

cmd/gateway-handlers.go

@@ -19,10 +19,12 @@ package cmd
 import (
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
+	"strconv"
 
+	"encoding/hex"
 	"encoding/json"
-	"encoding/xml"
 
 	router "github.com/gorilla/mux"
 	"github.com/minio/minio-go/pkg/policy"
@@ -52,17 +54,23 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
 		// Signature V2 validation.
 		s3Error := isReqAuthenticatedV2(r)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	case authTypeSigned, authTypePresigned:
 		s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
+	case authTypeAnonymous:
+		// No verification needed for anonymous requests.
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, ErrAccessDenied, r.URL)
+		return
 	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
@@ -132,7 +140,7 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	// Reads the object at startOffset and writes to mw.
-	if err := getObject(bucket, object, startOffset, length, writer); err != nil {
+	if err = getObject(bucket, object, startOffset, length, writer); err != nil {
 		errorIf(err, "Unable to write to client.")
 		if !dataWritten {
 			// Error response only if no data has been written to client yet. i.e if
@@ -149,6 +157,181 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
 		// call wrter.Write(nil) to set appropriate headers.
 		writer.Write(nil)
 	}
+
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		host, port = "", ""
+	}
+
+	// Notify object accessed via a GET request.
+	eventNotify(eventData{
+		Type:      ObjectAccessedGet,
+		Bucket:    bucket,
+		ObjInfo:   objInfo,
+		ReqParams: extractReqParams(r),
+		UserAgent: r.UserAgent(),
+		Host:      host,
+		Port:      port,
+	})
+}
+
+// PutObjectHandler - PUT Object
+// ----------
+// This implementation of the PUT operation adds an object to a bucket.
+func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// X-Amz-Copy-Source shouldn't be set for this call.
+	if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
+		writeErrorResponse(w, ErrInvalidCopySource, r.URL)
+		return
+	}
+
+	var object, bucket string
+	vars := router.Vars(r)
+	bucket = vars["bucket"]
+	object = vars["object"]
+
+	// TODO: we should validate the object name here
+
+	// Get Content-Md5 sent by client and verify if valid
+	md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5"))
+	if err != nil {
+		errorIf(err, "Unable to validate content-md5 format.")
+		writeErrorResponse(w, ErrInvalidDigest, r.URL)
+		return
+	}
+
+	/// if Content-Length is unknown/missing, deny the request
+	size := r.ContentLength
+	reqAuthType := getRequestAuthType(r)
+	if reqAuthType == authTypeStreamingSigned {
+		sizeStr := r.Header.Get("x-amz-decoded-content-length")
+		size, err = strconv.ParseInt(sizeStr, 10, 64)
+		if err != nil {
+			errorIf(err, "Unable to parse `x-amz-decoded-content-length` into its integer value", sizeStr)
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+	}
+	if size == -1 {
+		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		return
+	}
+
+	/// maximum Upload size for objects in a single operation
+	if isMaxObjectSize(size) {
+		writeErrorResponse(w, ErrEntityTooLarge, r.URL)
+		return
+	}
+
+	// Extract metadata to be saved from incoming HTTP header.
+	metadata, err := extractMetadataFromHeader(r.Header)
+	if err != nil {
+		errorIf(err, "found invalid http request header")
+		writeErrorResponse(w, ErrInternalError, r.URL)
+		return
+	}
+	if reqAuthType == authTypeStreamingSigned {
+		if contentEncoding, ok := metadata["content-encoding"]; ok {
+			contentEncoding = trimAwsChunkedContentEncoding(contentEncoding)
+			if contentEncoding != "" {
+				// Make sure to trim and save the content-encoding
+				// parameter for a streaming signature which is set
+				// to a custom value for example: "aws-chunked,gzip".
+				metadata["content-encoding"] = contentEncoding
+			} else {
+				// Trimmed content encoding is empty when the header
+				// value is set to "aws-chunked" only.
+
+				// Make sure to delete the content-encoding parameter
+				// for a streaming signature which is set to value
+				// for example: "aws-chunked"
+				delete(metadata, "content-encoding")
+			}
+		}
+	}
+
+	// Make sure we hex encode md5sum here.
+	metadata["etag"] = hex.EncodeToString(md5Bytes)
+
+	// Lock the object.
+	objectLock := globalNSMutex.NewNSLock(bucket, object)
+	objectLock.Lock()
+	defer objectLock.Unlock()
+
+	var objInfo ObjectInfo
+	switch reqAuthType {
+	case authTypeAnonymous:
+		// Create anonymous object.
+		objInfo, err = objectAPI.AnonPutObject(bucket, object, size, r.Body, metadata, "")
+	case authTypeStreamingSigned:
+		// Initialize stream signature verifier.
+		reader, s3Error := newSignV4ChunkedReader(r)
+		if s3Error != ErrNone {
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
+			writeErrorResponse(w, s3Error, r.URL)
+			return
+		}
+		objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata, "")
+	case authTypeSignedV2, authTypePresignedV2:
+		s3Error := isReqAuthenticatedV2(r)
+		if s3Error != ErrNone {
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
+			writeErrorResponse(w, s3Error, r.URL)
+			return
+		}
+		objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, "")
+	case authTypePresigned, authTypeSigned:
+		if s3Error := reqSignatureV4Verify(r, serverConfig.GetRegion()); s3Error != ErrNone {
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
+			writeErrorResponse(w, s3Error, r.URL)
+			return
+		}
+
+		sha256sum := ""
+		if !skipContentSha256Cksum(r) {
+			sha256sum = getContentSha256Cksum(r)
+		}
+
+		// Create object.
+		objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum)
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, ErrAccessDenied, r.URL)
+		return
+	}
+
+	if err != nil {
+		errorIf(err, "Unable to save an object %s", r.URL.Path)
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
+	writeSuccessResponseHeadersOnly(w)
+
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		host, port = "", ""
+	}
+
+	// Notify object created event.
+	eventNotify(eventData{
+		Type:      ObjectCreatedPut,
+		Bucket:    bucket,
+		ObjInfo:   objInfo,
+		ReqParams: extractReqParams(r),
+		UserAgent: r.UserAgent(),
+		Host:      host,
+		Port:      port,
+	})
 }
 
 // HeadObjectHandler - HEAD Object
@@ -173,17 +356,23 @@ func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.R
 		// Signature V2 validation.
 		s3Error := isReqAuthenticatedV2(r)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	case authTypeSigned, authTypePresigned:
 		s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
+	case authTypeAnonymous:
+		// No verification needed for anonymous requests.
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, ErrAccessDenied, r.URL)
+		return
 	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
@@ -211,97 +400,23 @@ func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.R
 
 	// Successful response.
 	w.WriteHeader(http.StatusOK)
-}
 
-// DeleteMultipleObjectsHandler - deletes multiple objects.
-func (api gatewayAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
-	vars := router.Vars(r)
-	bucket := vars["bucket"]
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
-		return
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		host, port = "", ""
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
-
-	// Content-Length is required and should be non-zero
-	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
-		return
-	}
-
-	// Content-Md5 is requied should be set
-	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(w, ErrMissingContentMD5, r.URL)
-		return
-	}
-
-	// Allocate incoming content length bytes.
-	deleteXMLBytes := make([]byte, r.ContentLength)
-
-	// Read incoming body XML bytes.
-	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
-		errorIf(err, "Unable to read HTTP body.")
-		writeErrorResponse(w, ErrInternalError, r.URL)
-		return
-	}
-
-	// Unmarshal list of keys to be deleted.
-	deleteObjects := &DeleteObjectsRequest{}
-	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
-		errorIf(err, "Unable to unmarshal delete objects request XML.")
-		writeErrorResponse(w, ErrMalformedXML, r.URL)
-		return
-	}
-
-	var dErrs = make([]error, len(deleteObjects.Objects))
-
-	// Delete all requested objects in parallel.
-	for index, object := range deleteObjects.Objects {
-		dErr := objectAPI.DeleteObject(bucket, object.ObjectName)
-		if dErr != nil {
-			dErrs[index] = dErr
-		}
-	}
-
-	// Collect deleted objects and errors if any.
-	var deletedObjects []ObjectIdentifier
-	var deleteErrors []DeleteError
-	for index, err := range dErrs {
-		object := deleteObjects.Objects[index]
-		// Success deleted objects are collected separately.
-		if err == nil {
-			deletedObjects = append(deletedObjects, object)
-			continue
-		}
-		if _, ok := errorCause(err).(ObjectNotFound); ok {
-			// If the object is not found it should be
-			// accounted as deleted as per S3 spec.
-			deletedObjects = append(deletedObjects, object)
-			continue
-		}
-		errorIf(err, "Unable to delete object. %s", object.ObjectName)
-		// Error during delete should be collected separately.
-		deleteErrors = append(deleteErrors, DeleteError{
-			Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
-			Message: errorCodeResponse[toAPIErrorCode(err)].Description,
-			Key:     object.ObjectName,
-		})
-	}
-
-	// Generate response
-	response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
-	encodedSuccessResponse := encodeResponse(response)
-
-	// Write success response.
-	writeSuccessResponseXML(w, encodedSuccessResponse)
+	// Notify object accessed via a HEAD request.
+	eventNotify(eventData{
+		Type:      ObjectAccessedHead,
+		Bucket:    bucket,
+		ObjInfo:   objInfo,
+		ReqParams: extractReqParams(r),
+		UserAgent: r.UserAgent(),
+		Host:      host,
+		Port:      port,
+	})
 }
 
 // PutBucketPolicyHandler - PUT Bucket policy
@@ -503,15 +618,6 @@ func (api gatewayAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// validating region here, because isValidLocationConstraint
-	// reads body which has been read already. So only validating
-	// region here.
-	serverRegion := serverConfig.GetRegion()
-	if serverRegion != location {
-		writeErrorResponse(w, ErrInvalidRegion, r.URL)
-		return
-	}
-
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
 	bucketLock.Lock()
 	defer bucketLock.Unlock()
@@ -581,28 +687,30 @@ func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *htt
 		// Signature V2 validation.
 		s3Error := isReqAuthenticatedV2(r)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	case authTypeSigned, authTypePresigned:
 		s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
-	}
-
-	// Extract all the litsObjectsV1 query params to their native values.
-	prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
-
-	// Validate all the query params before beginning to serve the request.
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+	case authTypeAnonymous:
+		// No verification needed for anonymous requests.
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, ErrAccessDenied, r.URL)
 		return
 	}
 
+	// Extract all the listObjectsV1 query params to their native
+	// values.  N B We delegate validation of params to respective
+	// gateway backends.
+	prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
+
 	listObjects := objectAPI.ListObjects
 	if reqAuthType == authTypeAnonymous {
 		listObjects = objectAPI.AnonListObjects
@@ -617,6 +725,87 @@ func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *htt
 		return
 	}
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
+	// Write success response.
+	writeSuccessResponseXML(w, encodeResponse(response))
+}
+
+// ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
+// --------------------------
+// This implementation of the GET operation returns some or all (up to 1000)
+// of the objects in a bucket. You can use the request parameters as selection
+// criteria to return a subset of the objects in a bucket.
+//
+// NOTE: It is recommended that this API to be used for application development.
+// Minio continues to support ListObjectsV1 for supporting legacy tools.
+func (api gatewayAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
+	vars := router.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	reqAuthType := getRequestAuthType(r)
+
+	switch reqAuthType {
+	case authTypePresignedV2, authTypeSignedV2:
+		// Signature V2 validation.
+		s3Error := isReqAuthenticatedV2(r)
+		if s3Error != ErrNone {
+			errorIf(errSignatureMismatch, dumpRequest(r))
+			writeErrorResponse(w, s3Error, r.URL)
+			return
+		}
+	case authTypeSigned, authTypePresigned:
+		s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
+		if s3Error != ErrNone {
+			errorIf(errSignatureMismatch, dumpRequest(r))
+			writeErrorResponse(w, s3Error, r.URL)
+			return
+		}
+	case authTypeAnonymous:
+		// No verification needed for anonymous requests.
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, ErrAccessDenied, r.URL)
+		return
+	}
+
+	// Extract all the listObjectsV2 query params to their native values.
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+
+	// In ListObjectsV2 'continuation-token' is the marker.
+	marker := token
+	// Check if 'continuation-token' is empty.
+	if token == "" {
+		// Then we need to use 'start-after' as marker instead.
+		marker = startAfter
+	}
+
+	listObjectsV2 := objectAPI.ListObjectsV2
+	if reqAuthType == authTypeAnonymous {
+		listObjectsV2 = objectAPI.AnonListObjectsV2
+	}
+
+	// Validate the query params before beginning to serve the request.
+	// fetch-owner is not validated since it is a boolean
+	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+	// Inititate a list objects operation based on the input params.
+	// On success would return back ListObjectsV2Info object to be
+	// serialized as XML and sent as S3 compatible response body.
+	listObjectsV2Info, err := listObjectsV2(bucket, prefix, token, fetchOwner, delimiter, maxKeys)
+	if err != nil {
+		errorIf(err, "Unable to list objects. Args to listObjectsV2 are bucket=%s, prefix=%s, token=%s, delimiter=%s", bucket, prefix, token, delimiter)
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.ContinuationToken, startAfter, delimiter, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))
@@ -645,17 +834,23 @@ func (api gatewayAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.R
 		// Signature V2 validation.
 		s3Error := isReqAuthenticatedV2(r)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	case authTypeSigned, authTypePresigned:
 		s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
+	case authTypeAnonymous:
+		// No verification needed for anonymous requests.
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, ErrAccessDenied, r.URL)
+		return
 	}
 
 	getBucketInfo := objectAPI.GetBucketInfo
@@ -691,7 +886,7 @@ func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r
 		// Signature V2 validation.
 		s3Error := isReqAuthenticatedV2(r)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
@@ -702,10 +897,16 @@ func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r
 			s3Error = isReqAuthenticated(r, serverConfig.GetRegion())
 		}
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
+	case authTypeAnonymous:
+		// No verification needed for anonymous requests.
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, ErrAccessDenied, r.URL)
+		return
 	}
 
 	getBucketInfo := objectAPI.GetBucketInfo

cmd/gateway-main.go

@@ -21,60 +21,145 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"os/signal"
+	"runtime"
 	"strings"
+	"syscall"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/cli"
+	miniohttp "github.com/minio/minio/pkg/http"
 )
 
-var gatewayTemplate = `NAME:
+const azureGatewayTemplate = `NAME:
   {{.HelpName}} - {{.Usage}}
 
 USAGE:
-  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} BACKEND [ENDPOINT]
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
 {{if .VisibleFlags}}
 FLAGS:
   {{range .VisibleFlags}}{{.}}
   {{end}}{{end}}
-BACKEND:
-  azure: Microsoft Azure Blob Storage. Default ENDPOINT is https://core.windows.net
-  s3: Amazon Simple Storage Service (S3). Default ENDPOINT is https://s3.amazonaws.com
+ENDPOINT:
+  Azure server endpoint. Default ENDPOINT is https://core.windows.net
 
 ENVIRONMENT VARIABLES:
   ACCESS:
-     MINIO_ACCESS_KEY: Username or access key of your storage backend.
-     MINIO_SECRET_KEY: Password or secret key of your storage backend.
+     MINIO_ACCESS_KEY: Username or access key of Azure storage.
+     MINIO_SECRET_KEY: Password or secret key of Azure storage.
+
+  BROWSER:
+     MINIO_BROWSER: To disable web browser access, set this value to "off".
 
 EXAMPLES:
   1. Start minio gateway server for Azure Blob Storage backend.
       $ export MINIO_ACCESS_KEY=azureaccountname
       $ export MINIO_SECRET_KEY=azureaccountkey
-      $ {{.HelpName}} azure
+      $ {{.HelpName}}
 
-  2. Start minio gateway server for AWS S3 backend.
-      $ export MINIO_ACCESS_KEY=accesskey
-      $ export MINIO_SECRET_KEY=secretkey
-      $ {{.HelpName}} s3
-
-  3. Start minio gateway server for S3 backend on custom endpoint.
-      $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
-      $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
-      $ {{.HelpName}} s3 https://play.minio.io:9000
+  2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
+      $ export MINIO_ACCESS_KEY=azureaccountname
+      $ export MINIO_SECRET_KEY=azureaccountkey
+      $ {{.HelpName}} https://azure.example.com
 `
 
-var gatewayCmd = cli.Command{
-	Name:               "gateway",
-	Usage:              "Start object storage gateway.",
-	Action:             gatewayMain,
-	CustomHelpTemplate: gatewayTemplate,
-	Flags: append(serverFlags,
-		cli.BoolFlag{
-			Name:  "quiet",
-			Usage: "Disable startup banner.",
-		},
-	),
-	HideHelpCommand: true,
-}
+const s3GatewayTemplate = `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
+{{if .VisibleFlags}}
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}{{end}}
+ENDPOINT:
+  S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com
+
+ENVIRONMENT VARIABLES:
+  ACCESS:
+     MINIO_ACCESS_KEY: Username or access key of S3 storage.
+     MINIO_SECRET_KEY: Password or secret key of S3 storage.
+
+  BROWSER:
+     MINIO_BROWSER: To disable web browser access, set this value to "off".
+
+EXAMPLES:
+  1. Start minio gateway server for AWS S3 backend.
+      $ export MINIO_ACCESS_KEY=accesskey
+      $ export MINIO_SECRET_KEY=secretkey
+      $ {{.HelpName}}
+
+  2. Start minio gateway server for S3 backend on custom endpoint.
+      $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
+      $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+      $ {{.HelpName}} https://play.minio.io:9000
+`
+
+const gcsGatewayTemplate = `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} PROJECTID
+{{if .VisibleFlags}}
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}{{end}}
+PROJECTID:
+  GCS project id, there are no defaults this is mandatory.
+
+ENVIRONMENT VARIABLES:
+  ACCESS:
+     MINIO_ACCESS_KEY: Username or access key of GCS.
+     MINIO_SECRET_KEY: Password or secret key of GCS.
+
+  BROWSER:
+     MINIO_BROWSER: To disable web browser access, set this value to "off".
+
+EXAMPLES:
+  1. Start minio gateway server for GCS backend.
+      $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
+      (Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials)
+      $ export MINIO_ACCESS_KEY=accesskey
+      $ export MINIO_SECRET_KEY=secretkey
+      $ {{.HelpName}} mygcsprojectid
+
+`
+
+var (
+	azureBackendCmd = cli.Command{
+		Name:               "azure",
+		Usage:              "Microsoft Azure Blob Storage.",
+		Action:             azureGatewayMain,
+		CustomHelpTemplate: azureGatewayTemplate,
+		Flags:              append(serverFlags, globalFlags...),
+		HideHelpCommand:    true,
+	}
+
+	s3BackendCmd = cli.Command{
+		Name:               "s3",
+		Usage:              "Amazon Simple Storage Service (S3).",
+		Action:             s3GatewayMain,
+		CustomHelpTemplate: s3GatewayTemplate,
+		Flags:              append(serverFlags, globalFlags...),
+		HideHelpCommand:    true,
+	}
+	gcsBackendCmd = cli.Command{
+		Name:               "gcs",
+		Usage:              "Google Cloud Storage.",
+		Action:             gcsGatewayMain,
+		CustomHelpTemplate: gcsGatewayTemplate,
+		Flags:              append(serverFlags, globalFlags...),
+		HideHelpCommand:    true,
+	}
+
+	gatewayCmd = cli.Command{
+		Name:            "gateway",
+		Usage:           "Start object storage gateway.",
+		Flags:           append(serverFlags, globalFlags...),
+		HideHelpCommand: true,
+		Subcommands:     []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd},
+	}
+)
 
 // Represents the type of the gateway backend.
 type gatewayBackend string
@@ -82,63 +167,33 @@ type gatewayBackend string
 const (
 	azureBackend gatewayBackend = "azure"
 	s3Backend    gatewayBackend = "s3"
+	gcsBackend   gatewayBackend = "gcs"
 	// Add more backends here.
 )
 
-// Returns access and secretkey set from environment variables.
-func mustGetGatewayCredsFromEnv() (accessKey, secretKey string) {
-	// Fetch access keys from environment variables.
-	accessKey = os.Getenv("MINIO_ACCESS_KEY")
-	secretKey = os.Getenv("MINIO_SECRET_KEY")
-	if accessKey == "" || secretKey == "" {
-		fatalIf(errors.New("Missing credentials"), "Access and secret keys are mandatory to run Minio gateway server.")
-	}
-	return accessKey, secretKey
-}
-
 // Initialize gateway layer depending on the backend type.
 // Supported backend types are
 //
 // - Azure Blob Storage.
+// - AWS S3.
+// - Google Cloud Storage.
 // - Add your favorite backend here.
-func newGatewayLayer(backendType, endpoint, accessKey, secretKey string, secure bool) (GatewayLayer, error) {
-	switch gatewayBackend(backendType) {
+func newGatewayLayer(backendType gatewayBackend, arg string) (GatewayLayer, error) {
+	switch backendType {
 	case azureBackend:
-		return newAzureLayer(endpoint, accessKey, secretKey, secure)
+		return newAzureLayer(arg)
 	case s3Backend:
-		return newS3Gateway(endpoint, accessKey, secretKey, secure)
+		return newS3Gateway(arg)
+	case gcsBackend:
+		// FIXME: The following print command is temporary and
+		// will be removed when gcs is ready for production use.
+		log.Println(colorYellow("\n               *** Warning: Not Ready for Production ***"))
+		return newGCSGateway(arg)
 	}
 
 	return nil, fmt.Errorf("Unrecognized backend type %s", backendType)
 }
 
-// Initialize a new gateway config.
-//
-// DO NOT save this config, this is meant to be
-// only used in memory.
-func newGatewayConfig(accessKey, secretKey, region string) error {
-	// Initialize server config.
-	srvCfg := newServerConfigV18()
-
-	// If env is set for a fresh start, save them to config file.
-	srvCfg.SetCredential(credential{
-		AccessKey: accessKey,
-		SecretKey: secretKey,
-	})
-
-	// Set custom region.
-	srvCfg.SetRegion(region)
-
-	// hold the mutex lock before a new config is assigned.
-	// Save the new config globally.
-	// unlock the mutex.
-	serverConfigMu.Lock()
-	serverConfig = srvCfg
-	serverConfigMu.Unlock()
-
-	return nil
-}
-
 // Return endpoint.
 func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error) {
 	schemeSpecified := len(strings.Split(arg, "://")) > 1
@@ -162,45 +217,127 @@ func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error)
 	}
 }
 
-// Handler for 'minio gateway'.
-func gatewayMain(ctx *cli.Context) {
-	if !ctx.Args().Present() || ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, "gateway", 1)
+// Validate gateway arguments.
+func validateGatewayArguments(serverAddr, endpointAddr string) error {
+	if err := CheckLocalServerAddr(serverAddr); err != nil {
+		return err
 	}
 
-	// Fetch access and secret key from env.
-	accessKey, secretKey := mustGetGatewayCredsFromEnv()
+	if runtime.GOOS == "darwin" {
+		_, port := mustSplitHostPort(serverAddr)
+		// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
+		// to IPv6 address i.e minio will start listening on IPv6 address whereas another
+		// (non-)minio process is listening on IPv4 of given port.
+		// To avoid this error situation we check for port availability only for macOS.
+		if err := checkPortAvailability(port); err != nil {
+			return err
+		}
+	}
 
-	// Initialize new gateway config.
-	//
-	// TODO: add support for custom region when we add
-	// support for S3 backend storage, currently this can
-	// default to "us-east-1"
-	newGatewayConfig(accessKey, secretKey, globalMinioDefaultRegion)
+	if endpointAddr != "" {
+		// Reject the endpoint if it points to the gateway handler itself.
+		sameTarget, err := sameLocalAddrs(endpointAddr, serverAddr)
+		if err != nil {
+			return err
+		}
+		if sameTarget {
+			return errors.New("endpoint points to the local gateway")
+		}
+	}
+	return nil
+}
 
+// Handler for 'minio gateway azure' command line.
+func azureGatewayMain(ctx *cli.Context) {
+	if ctx.Args().Present() && ctx.Args().First() == "help" {
+		cli.ShowCommandHelpAndExit(ctx, "azure", 1)
+	}
+
+	// Validate gateway arguments.
+	fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument")
+
+	gatewayMain(ctx, azureBackend)
+}
+
+// Handler for 'minio gateway s3' command line.
+func s3GatewayMain(ctx *cli.Context) {
+	if ctx.Args().Present() && ctx.Args().First() == "help" {
+		cli.ShowCommandHelpAndExit(ctx, "s3", 1)
+	}
+
+	// Validate gateway arguments.
+	fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument")
+
+	gatewayMain(ctx, s3Backend)
+}
+
+// Handler for 'minio gateway gcs' command line
+func gcsGatewayMain(ctx *cli.Context) {
+	if ctx.Args().Present() && ctx.Args().First() == "help" {
+		cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
+	}
+
+	if !isValidGCSProjectIDFormat(ctx.Args().First()) {
+		errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First())
+		cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
+	}
+
+	gatewayMain(ctx, gcsBackend)
+}
+
+// Handler for 'minio gateway'.
+func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
 	// Get quiet flag from command line argument.
 	quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
 	if quietFlag {
 		log.EnableQuiet()
 	}
 
-	// First argument is selected backend type.
-	backendType := ctx.Args().First()
+	// Fetch address option
+	gatewayAddr := ctx.GlobalString("address")
+	if gatewayAddr == ":"+globalMinioPort {
+		gatewayAddr = ctx.String("address")
+	}
 
-	// Second argument is endpoint.	If no endpoint is specified then the
-	// gateway implementation should use a default setting.
-	endPoint, secure, err := parseGatewayEndpoint(ctx.Args().Get(1))
-	fatalIf(err, "Unable to parse endpoint")
+	// Handle common command args.
+	handleCommonCmdArgs(ctx)
 
-	// Create certs path for SSL configuration.
-	fatalIf(createConfigDir(), "Unable to create configuration directory")
+	// Handle common env vars.
+	handleCommonEnvVars()
 
-	newObject, err := newGatewayLayer(backendType, endPoint, accessKey, secretKey, secure)
-	fatalIf(err, "Unable to initialize gateway layer")
+	// Validate if we have access, secret set through environment.
+	if !globalIsEnvCreds {
+		fatalIf(fmt.Errorf("Access and Secret keys should be set through ENVs for backend [%s]", backendType), "")
+	}
+
+	// Create certs path.
+	fatalIf(createConfigDir(), "Unable to create configuration directories.")
+
+	// Initialize gateway config.
+	initConfig()
+
+	// Enable loggers as per configuration file.
+	enableLoggers()
+
+	// Init the error tracing module.
+	initError()
+
+	// Check and load SSL certificates.
+	var err error
+	globalPublicCerts, globalRootCAs, globalTLSCertificate, globalIsSSL, err = getSSLConfig()
+	fatalIf(err, "Invalid SSL certificate file")
 
 	initNSLock(false) // Enable local namespace lock.
 
+	newObject, err := newGatewayLayer(backendType, ctx.Args().First())
+	fatalIf(err, "Unable to initialize gateway layer")
+
 	router := mux.NewRouter().SkipClean(true)
+
+	// Register web router when its enabled.
+	if globalIsBrowserEnabled {
+		fatalIf(registerWebRouter(router), "Unable to configure web browser")
+	}
 	registerGatewayAPIRouter(router, newObject)
 
 	var handlerFns = []HandlerFunc{
@@ -211,6 +348,13 @@ func gatewayMain(ctx *cli.Context) {
 		// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
 		setCrossDomainPolicy,
 		// Validates all incoming requests to have a valid date header.
+		// Redirect some pre-defined browser request paths to a static location prefix.
+		setBrowserRedirectHandler,
+		// Validates if incoming request is for restricted buckets.
+		setPrivateBucketHandler,
+		// Adds cache control for all browser requests.
+		setBrowserCacheControlHandler,
+		// Validates all incoming requests to have a valid date header.
 		setTimeValidityHandler,
 		// CORS setting for all browser API requests.
 		setCorsHandler,
@@ -221,24 +365,19 @@ func gatewayMain(ctx *cli.Context) {
 		// routes them accordingly. Client receives a HTTP error for
 		// invalid/unsupported signatures.
 		setAuthHandler,
+		// Add new handlers here.
+
 	}
 
-	apiServer := NewServerMux(ctx.String("address"), registerHandlers(router, handlerFns...))
-
-	_, _, globalIsSSL, err = getSSLConfig()
-	fatalIf(err, "Invalid SSL key file")
+	globalHTTPServer = miniohttp.NewServer([]string{gatewayAddr}, registerHandlers(router, handlerFns...), globalTLSCertificate)
 
 	// Start server, automatically configures TLS if certs are available.
 	go func() {
-		cert, key := "", ""
-		if globalIsSSL {
-			cert, key = getPublicCertFile(), getPrivateKeyFile()
-		}
-
-		aerr := apiServer.ListenAndServe(cert, key)
-		fatalIf(aerr, "Failed to start minio server")
+		globalHTTPServerErrorCh <- globalHTTPServer.Start()
 	}()
 
+	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
+
 	// Once endpoints are finalized, initialize the new object api.
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject
@@ -247,15 +386,21 @@ func gatewayMain(ctx *cli.Context) {
 	// Prints the formatted startup message once object layer is initialized.
 	if !quietFlag {
 		mode := ""
-		if gatewayBackend(backendType) == azureBackend {
+		switch gatewayBackend(backendType) {
+		case azureBackend:
 			mode = globalMinioModeGatewayAzure
-		} else if gatewayBackend(backendType) == s3Backend {
+		case gcsBackend:
+			mode = globalMinioModeGatewayGCS
+		case s3Backend:
 			mode = globalMinioModeGatewayS3
 		}
+
+		// Check update mode.
 		checkUpdate(mode)
-		apiEndpoints := getAPIEndpoints(apiServer.Addr)
-		printGatewayStartupMessage(apiEndpoints, accessKey, secretKey, backendType)
+
+		// Print gateway startup message.
+		printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), backendType)
 	}
 
-	<-globalServiceDoneCh
+	handleSignals()
 }

cmd/gateway-main_test.go

@@ -16,7 +16,10 @@
 
 package cmd
 
-import "testing"
+import (
+	"strings"
+	"testing"
+)
 
 // Test parseGatewayEndpoint
 func TestParseGatewayEndpoint(t *testing.T) {
@@ -48,3 +51,35 @@ func TestParseGatewayEndpoint(t *testing.T) {
 		}
 	}
 }
+
+// Test validateGatewayArguments
+func TestValidateGatewayArguments(t *testing.T) {
+	nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool {
+		return !strings.HasPrefix(ip, "127.")
+	}, "")
+	if len(nonLoopBackIPs) == 0 {
+		t.Fatalf("No non-loop back IP address found for this host")
+	}
+	nonLoopBackIP := nonLoopBackIPs.ToSlice()[0]
+
+	testCases := []struct {
+		serverAddr   string
+		endpointAddr string
+		valid        bool
+	}{
+		{":9000", "http://localhost:9001", true},
+		{":9000", "http://google.com", true},
+		{"123.123.123.123:9000", "http://localhost:9000", false},
+		{":9000", "http://localhost:9000", false},
+		{":9000", nonLoopBackIP + ":9000", false},
+	}
+	for i, test := range testCases {
+		err := validateGatewayArguments(test.serverAddr, test.endpointAddr)
+		if test.valid && err != nil {
+			t.Errorf("Test %d expected not to return error but got %s", i+1, err)
+		}
+		if !test.valid && err == nil {
+			t.Errorf("Test %d expected to fail but it did not", i+1)
+		}
+	}
+}

cmd/gateway-router.go

@@ -27,14 +27,17 @@ import (
 type GatewayLayer interface {
 	ObjectLayer
 
-	MakeBucketWithLocation(bucket, location string) error
-
 	AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
 	AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
+
+	AnonPutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error)
+
 	SetBucketPolicies(string, policy.BucketAccessPolicy) error
 	GetBucketPolicies(string) (policy.BucketAccessPolicy, error)
 	DeleteBucketPolicies(string) error
 	AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
+	AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error)
+	ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error)
 	AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error)
 }
 

cmd/gateway-s3-anonymous.go

@@ -17,13 +17,44 @@
 package cmd
 
 import (
+	"encoding/hex"
 	"io"
 
 	minio "github.com/minio/minio-go"
 )
 
+// AnonPutObject creates a new object anonymously with the incoming data,
+func (l *s3Objects) AnonPutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, e error) {
+	var sha256sumBytes []byte
+
+	var err error
+	if sha256sum != "" {
+		sha256sumBytes, err = hex.DecodeString(sha256sum)
+		if err != nil {
+			return objInfo, s3ToObjectError(traceError(err), bucket, object)
+		}
+	}
+
+	var md5sumBytes []byte
+	md5sum := metadata["etag"]
+	if md5sum != "" {
+		md5sumBytes, err = hex.DecodeString(md5sum)
+		if err != nil {
+			return objInfo, s3ToObjectError(traceError(err), bucket, object)
+		}
+		delete(metadata, "etag")
+	}
+
+	oi, err := l.anonClient.PutObject(bucket, object, size, data, md5sumBytes, sha256sumBytes, toMinioClientMetadata(metadata))
+	if err != nil {
+		return objInfo, s3ToObjectError(traceError(err), bucket, object)
+	}
+
+	return fromMinioClientObjectInfo(bucket, oi), nil
+}
+
 // AnonGetObject - Get object anonymously
-func (l *s3Gateway) AnonGetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
+func (l *s3Objects) AnonGetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
 	r := minio.NewGetReqHeaders()
 	if err := r.SetRange(startOffset, startOffset+length-1); err != nil {
 		return s3ToObjectError(traceError(err), bucket, key)
@@ -43,37 +74,47 @@ func (l *s3Gateway) AnonGetObject(bucket string, key string, startOffset int64,
 }
 
 // AnonGetObjectInfo - Get object info anonymously
-func (l *s3Gateway) AnonGetObjectInfo(bucket string, object string) (ObjectInfo, error) {
+func (l *s3Objects) AnonGetObjectInfo(bucket string, object string) (objInfo ObjectInfo, e error) {
 	r := minio.NewHeadReqHeaders()
 	oi, err := l.anonClient.StatObject(bucket, object, r)
 	if err != nil {
-		return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
+		return objInfo, s3ToObjectError(traceError(err), bucket, object)
 	}
 
 	return fromMinioClientObjectInfo(bucket, oi), nil
 }
 
 // AnonListObjects - List objects anonymously
-func (l *s3Gateway) AnonListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+func (l *s3Objects) AnonListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
 	result, err := l.anonClient.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		return ListObjectsInfo{}, s3ToObjectError(traceError(err), bucket)
+		return loi, s3ToObjectError(traceError(err), bucket)
 	}
 
 	return fromMinioClientListBucketResult(bucket, result), nil
 }
 
+// AnonListObjectsV2 - List objects in V2 mode, anonymously
+func (l *s3Objects) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (loi ListObjectsV2Info, e error) {
+	result, err := l.anonClient.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
+	if err != nil {
+		return loi, s3ToObjectError(traceError(err), bucket)
+	}
+
+	return fromMinioClientListBucketV2Result(bucket, result), nil
+}
+
 // AnonGetBucketInfo - Get bucket metadata anonymously.
-func (l *s3Gateway) AnonGetBucketInfo(bucket string) (BucketInfo, error) {
+func (l *s3Objects) AnonGetBucketInfo(bucket string) (bi BucketInfo, e error) {
 	if exists, err := l.anonClient.BucketExists(bucket); err != nil {
-		return BucketInfo{}, s3ToObjectError(traceError(err), bucket)
+		return bi, s3ToObjectError(traceError(err), bucket)
 	} else if !exists {
-		return BucketInfo{}, traceError(BucketNotFound{Bucket: bucket})
+		return bi, traceError(BucketNotFound{Bucket: bucket})
 	}
 
 	buckets, err := l.anonClient.ListBuckets()
 	if err != nil {
-		return BucketInfo{}, s3ToObjectError(traceError(err), bucket)
+		return bi, s3ToObjectError(traceError(err), bucket)
 	}
 
 	for _, bi := range buckets {
@@ -87,5 +128,5 @@ func (l *s3Gateway) AnonGetBucketInfo(bucket string) (BucketInfo, error) {
 		}, nil
 	}
 
-	return BucketInfo{}, traceError(BucketNotFound{Bucket: bucket})
+	return bi, traceError(BucketNotFound{Bucket: bucket})
 }

cmd/gateway-s3-unsupported.go

@@ -17,26 +17,26 @@
 package cmd
 
 // HealBucket - Not relevant.
-func (l *s3Gateway) HealBucket(bucket string) error {
+func (l *s3Objects) HealBucket(bucket string) error {
 	return traceError(NotImplemented{})
 }
 
 // ListBucketsHeal - Not relevant.
-func (l *s3Gateway) ListBucketsHeal() (buckets []BucketInfo, err error) {
+func (l *s3Objects) ListBucketsHeal() (buckets []BucketInfo, err error) {
 	return []BucketInfo{}, traceError(NotImplemented{})
 }
 
 // HealObject - Not relevant.
-func (l *s3Gateway) HealObject(bucket string, object string) (int, int, error) {
+func (l *s3Objects) HealObject(bucket string, object string) (int, int, error) {
 	return 0, 0, traceError(NotImplemented{})
 }
 
 // ListObjectsHeal - Not relevant.
-func (l *s3Gateway) ListObjectsHeal(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
-	return ListObjectsInfo{}, traceError(NotImplemented{})
+func (l *s3Objects) ListObjectsHeal(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
+	return loi, traceError(NotImplemented{})
 }
 
 // ListUploadsHeal - Not relevant.
-func (l *s3Gateway) ListUploadsHeal(bucket string, prefix string, marker string, uploadIDMarker string, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
-	return ListMultipartsInfo{}, traceError(NotImplemented{})
+func (l *s3Objects) ListUploadsHeal(bucket string, prefix string, marker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+	return lmi, traceError(NotImplemented{})
 }

cmd/gateway-s3.go

@@ -91,21 +91,37 @@ func s3ToObjectError(err error, params ...string) error {
 	return e
 }
 
-// s3Gateway implements gateway for Minio and S3 compatible object storage servers.
-type s3Gateway struct {
+// s3Objects implements gateway for Minio and S3 compatible object storage servers.
+type s3Objects struct {
 	Client     *minio.Core
 	anonClient *minio.Core
 }
 
 // newS3Gateway returns s3 gatewaylayer
-func newS3Gateway(endpoint string, accessKey, secretKey string, secure bool) (GatewayLayer, error) {
-	if endpoint == "" {
-		endpoint = "s3.amazonaws.com"
-		secure = true
+func newS3Gateway(host string) (GatewayLayer, error) {
+
+	var err error
+	var endpoint string
+	var secure = true
+
+	// Validate host parameters.
+	if host != "" {
+		// Override default params if the host is provided
+		endpoint, secure, err = parseGatewayEndpoint(host)
+		if err != nil {
+			return nil, err
+		}
 	}
 
+	// Default endpoint parameters
+	if endpoint == "" {
+		endpoint = "s3.amazonaws.com"
+	}
+
+	creds := serverConfig.GetCredential()
+
 	// Initialize minio client object.
-	client, err := minio.NewCore(endpoint, accessKey, secretKey, secure)
+	client, err := minio.NewCore(endpoint, creds.AccessKey, creds.SecretKey, secure)
 	if err != nil {
 		return nil, err
 	}
@@ -115,7 +131,7 @@ func newS3Gateway(endpoint string, accessKey, secretKey string, secure bool) (Ga
 		return nil, err
 	}
 
-	return &s3Gateway{
+	return &s3Objects{
 		Client:     client,
 		anonClient: anonClient,
 	}, nil
@@ -123,24 +139,18 @@ func newS3Gateway(endpoint string, accessKey, secretKey string, secure bool) (Ga
 
 // Shutdown saves any gateway metadata to disk
 // if necessary and reload upon next restart.
-func (l *s3Gateway) Shutdown() error {
+func (l *s3Objects) Shutdown() error {
 	// TODO
 	return nil
 }
 
 // StorageInfo is not relevant to S3 backend.
-func (l *s3Gateway) StorageInfo() StorageInfo {
-	return StorageInfo{}
+func (l *s3Objects) StorageInfo() (si StorageInfo) {
+	return si
 }
 
 // MakeBucket creates a new container on S3 backend.
-func (l *s3Gateway) MakeBucket(bucket string) error {
-	// will never be called, only satisfy ObjectLayer interface
-	return traceError(NotImplemented{})
-}
-
-// MakeBucket creates a new container on S3 backend.
-func (l *s3Gateway) MakeBucketWithLocation(bucket, location string) error {
+func (l *s3Objects) MakeBucketWithLocation(bucket, location string) error {
 	err := l.Client.MakeBucket(bucket, location)
 	if err != nil {
 		return s3ToObjectError(traceError(err), bucket)
@@ -149,10 +159,10 @@ func (l *s3Gateway) MakeBucketWithLocation(bucket, location string) error {
 }
 
 // GetBucketInfo gets bucket metadata..
-func (l *s3Gateway) GetBucketInfo(bucket string) (BucketInfo, error) {
+func (l *s3Objects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
 	buckets, err := l.Client.ListBuckets()
 	if err != nil {
-		return BucketInfo{}, s3ToObjectError(traceError(err), bucket)
+		return bi, s3ToObjectError(traceError(err), bucket)
 	}
 
 	for _, bi := range buckets {
@@ -166,11 +176,11 @@ func (l *s3Gateway) GetBucketInfo(bucket string) (BucketInfo, error) {
 		}, nil
 	}
 
-	return BucketInfo{}, traceError(BucketNotFound{Bucket: bucket})
+	return bi, traceError(BucketNotFound{Bucket: bucket})
 }
 
 // ListBuckets lists all S3 buckets
-func (l *s3Gateway) ListBuckets() ([]BucketInfo, error) {
+func (l *s3Objects) ListBuckets() ([]BucketInfo, error) {
 	buckets, err := l.Client.ListBuckets()
 	if err != nil {
 		return nil, err
@@ -188,7 +198,7 @@ func (l *s3Gateway) ListBuckets() ([]BucketInfo, error) {
 }
 
 // DeleteBucket deletes a bucket on S3
-func (l *s3Gateway) DeleteBucket(bucket string) error {
+func (l *s3Objects) DeleteBucket(bucket string) error {
 	err := l.Client.RemoveBucket(bucket)
 	if err != nil {
 		return s3ToObjectError(traceError(err), bucket)
@@ -197,20 +207,20 @@ func (l *s3Gateway) DeleteBucket(bucket string) error {
 }
 
 // ListObjects lists all blobs in S3 bucket filtered by prefix
-func (l *s3Gateway) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+func (l *s3Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
 	result, err := l.Client.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		return ListObjectsInfo{}, s3ToObjectError(traceError(err), bucket)
+		return loi, s3ToObjectError(traceError(err), bucket)
 	}
 
 	return fromMinioClientListBucketResult(bucket, result), nil
 }
 
 // ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
-func (l *s3Gateway) ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (ListObjectsV2Info, error) {
+func (l *s3Objects) ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (loi ListObjectsV2Info, e error) {
 	result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
 	if err != nil {
-		return ListObjectsV2Info{}, s3ToObjectError(traceError(err), bucket)
+		return loi, s3ToObjectError(traceError(err), bucket)
 	}
 
 	return fromMinioClientListBucketV2Result(bucket, result), nil
@@ -266,10 +276,17 @@ func fromMinioClientListBucketResult(bucket string, result minio.ListBucketResul
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *s3Gateway) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
+func (l *s3Objects) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
 	r := minio.NewGetReqHeaders()
-	if err := r.SetRange(startOffset, startOffset+length-1); err != nil {
-		return s3ToObjectError(traceError(err), bucket, key)
+
+	if length < 0 && length != -1 {
+		return s3ToObjectError(traceError(errInvalidArgument), bucket, key)
+	}
+
+	if startOffset >= 0 && length >= 0 {
+		if err := r.SetRange(startOffset, startOffset+length-1); err != nil {
+			return s3ToObjectError(traceError(err), bucket, key)
+		}
 	}
 	object, _, err := l.Client.GetObject(bucket, key, r)
 	if err != nil {
@@ -278,10 +295,9 @@ func (l *s3Gateway) GetObject(bucket string, key string, startOffset int64, leng
 
 	defer object.Close()
 
-	if _, err := io.CopyN(writer, object, length); err != nil {
+	if _, err := io.Copy(writer, object); err != nil {
 		return s3ToObjectError(traceError(err), bucket, key)
 	}
-
 	return nil
 }
 
@@ -295,7 +311,7 @@ func fromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 		Name:            oi.Key,
 		ModTime:         oi.LastModified,
 		Size:            oi.Size,
-		MD5Sum:          oi.ETag,
+		ETag:            canonicalizeETag(oi.ETag),
 		UserDefined:     userDefined,
 		ContentType:     oi.ContentType,
 		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
@@ -303,7 +319,7 @@ func fromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo
-func (l *s3Gateway) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
+func (l *s3Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
 	r := minio.NewHeadReqHeaders()
 	oi, err := l.Client.StatObject(bucket, object, r)
 	if err != nil {
@@ -314,52 +330,52 @@ func (l *s3Gateway) GetObjectInfo(bucket string, object string) (objInfo ObjectI
 }
 
 // PutObject creates a new object with the incoming data,
-func (l *s3Gateway) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error) {
+func (l *s3Objects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, e error) {
 	var sha256sumBytes []byte
 
 	var err error
 	if sha256sum != "" {
 		sha256sumBytes, err = hex.DecodeString(sha256sum)
 		if err != nil {
-			return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
+			return objInfo, s3ToObjectError(traceError(err), bucket, object)
 		}
 	}
 
 	var md5sumBytes []byte
-	md5sum := metadata["md5Sum"]
+	md5sum := metadata["etag"]
 	if md5sum != "" {
 		md5sumBytes, err = hex.DecodeString(md5sum)
 		if err != nil {
-			return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
+			return objInfo, s3ToObjectError(traceError(err), bucket, object)
 		}
-		delete(metadata, "md5Sum")
+		delete(metadata, "etag")
 	}
 
 	oi, err := l.Client.PutObject(bucket, object, size, data, md5sumBytes, sha256sumBytes, toMinioClientMetadata(metadata))
 	if err != nil {
-		return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
+		return objInfo, s3ToObjectError(traceError(err), bucket, object)
 	}
 
 	return fromMinioClientObjectInfo(bucket, oi), nil
 }
 
 // CopyObject copies a blob from source container to destination container.
-func (l *s3Gateway) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string, metadata map[string]string) (ObjectInfo, error) {
+func (l *s3Objects) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string, metadata map[string]string) (objInfo ObjectInfo, e error) {
 	err := l.Client.CopyObject(destBucket, destObject, path.Join(srcBucket, srcObject), minio.CopyConditions{})
 	if err != nil {
-		return ObjectInfo{}, s3ToObjectError(traceError(err), srcBucket, srcObject)
+		return objInfo, s3ToObjectError(traceError(err), srcBucket, srcObject)
 	}
 
 	oi, err := l.GetObjectInfo(destBucket, destObject)
 	if err != nil {
-		return ObjectInfo{}, s3ToObjectError(traceError(err), destBucket, destObject)
+		return objInfo, s3ToObjectError(traceError(err), destBucket, destObject)
 	}
 
 	return oi, nil
 }
 
 // DeleteObject deletes a blob in bucket
-func (l *s3Gateway) DeleteObject(bucket string, object string) error {
+func (l *s3Objects) DeleteObject(bucket string, object string) error {
 	err := l.Client.RemoveObject(bucket, object)
 	if err != nil {
 		return s3ToObjectError(traceError(err), bucket, object)
@@ -407,10 +423,10 @@ func fromMinioClientListMultipartsInfo(lmur minio.ListMultipartUploadsResult) Li
 }
 
 // ListMultipartUploads lists all multipart uploads.
-func (l *s3Gateway) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+func (l *s3Objects) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
 	result, err := l.Client.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
-		return ListMultipartsInfo{}, err
+		return lmi, err
 	}
 
 	return fromMinioClientListMultipartsInfo(result), nil
@@ -435,12 +451,12 @@ func toMinioClientMetadata(metadata map[string]string) map[string][]string {
 }
 
 // NewMultipartUpload upload object in multiple parts
-func (l *s3Gateway) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) {
+func (l *s3Objects) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) {
 	return l.Client.NewMultipartUpload(bucket, object, toMinioClientMetadata(metadata))
 }
 
 // CopyObjectPart copy part of object to other bucket and object
-func (l *s3Gateway) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
+func (l *s3Objects) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
 	// FIXME: implement CopyObjectPart
 	return PartInfo{}, traceError(NotImplemented{})
 }
@@ -449,27 +465,27 @@ func (l *s3Gateway) CopyObjectPart(srcBucket string, srcObject string, destBucke
 func fromMinioClientObjectPart(op minio.ObjectPart) PartInfo {
 	return PartInfo{
 		Size:         op.Size,
-		ETag:         op.ETag,
+		ETag:         canonicalizeETag(op.ETag),
 		LastModified: op.LastModified,
 		PartNumber:   op.PartNumber,
 	}
 }
 
 // PutObjectPart puts a part of object in bucket
-func (l *s3Gateway) PutObjectPart(bucket string, object string, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (PartInfo, error) {
+func (l *s3Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (pi PartInfo, e error) {
 	md5HexBytes, err := hex.DecodeString(md5Hex)
 	if err != nil {
-		return PartInfo{}, err
+		return pi, err
 	}
 
 	sha256sumBytes, err := hex.DecodeString(sha256sum)
 	if err != nil {
-		return PartInfo{}, err
+		return pi, err
 	}
 
 	info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, size, data, md5HexBytes, sha256sumBytes)
 	if err != nil {
-		return PartInfo{}, err
+		return pi, err
 	}
 
 	return fromMinioClientObjectPart(info), nil
@@ -501,17 +517,17 @@ func fromMinioClientListPartsInfo(lopr minio.ListObjectPartsResult) ListPartsInf
 }
 
 // ListObjectParts returns all object parts for specified object in specified bucket
-func (l *s3Gateway) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (ListPartsInfo, error) {
+func (l *s3Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, e error) {
 	result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
 	if err != nil {
-		return ListPartsInfo{}, err
+		return lpi, err
 	}
 
 	return fromMinioClientListPartsInfo(result), nil
 }
 
 // AbortMultipartUpload aborts a ongoing multipart upload
-func (l *s3Gateway) AbortMultipartUpload(bucket string, object string, uploadID string) error {
+func (l *s3Objects) AbortMultipartUpload(bucket string, object string, uploadID string) error {
 	return l.Client.AbortMultipartUpload(bucket, object, uploadID)
 }
 
@@ -533,17 +549,17 @@ func toMinioClientCompleteParts(parts []completePart) []minio.CompletePart {
 }
 
 // CompleteMultipartUpload completes ongoing multipart upload and finalizes object
-func (l *s3Gateway) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []completePart) (ObjectInfo, error) {
+func (l *s3Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []completePart) (oi ObjectInfo, e error) {
 	err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, toMinioClientCompleteParts(uploadedParts))
 	if err != nil {
-		return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
+		return oi, s3ToObjectError(traceError(err), bucket, object)
 	}
 
 	return l.GetObjectInfo(bucket, object)
 }
 
 // SetBucketPolicies sets policy on bucket
-func (l *s3Gateway) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
+func (l *s3Objects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
 	if err := l.Client.PutBucketPolicy(bucket, policyInfo); err != nil {
 		return s3ToObjectError(traceError(err), bucket, "")
 	}
@@ -552,7 +568,7 @@ func (l *s3Gateway) SetBucketPolicies(bucket string, policyInfo policy.BucketAcc
 }
 
 // GetBucketPolicies will get policy on bucket
-func (l *s3Gateway) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
+func (l *s3Objects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
 	policyInfo, err := l.Client.GetBucketPolicy(bucket)
 	if err != nil {
 		return policy.BucketAccessPolicy{}, s3ToObjectError(traceError(err), bucket, "")
@@ -561,10 +577,9 @@ func (l *s3Gateway) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy,
 }
 
 // DeleteBucketPolicies deletes all policies on bucket
-func (l *s3Gateway) DeleteBucketPolicies(bucket string) error {
+func (l *s3Objects) DeleteBucketPolicies(bucket string) error {
 	if err := l.Client.PutBucketPolicy(bucket, policy.BucketAccessPolicy{}); err != nil {
 		return s3ToObjectError(traceError(err), bucket, "")
 	}
-
 	return nil
 }

cmd/gateway-startup-msg.go

@@ -18,28 +18,19 @@ package cmd
 
 import (
 	"fmt"
-	"runtime"
 	"strings"
 )
 
 // Prints the formatted startup message.
-func printGatewayStartupMessage(apiEndPoints []string, accessKey, secretKey, backendType string) {
+func printGatewayStartupMessage(apiEndPoints []string, backendType gatewayBackend) {
+	strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
+
 	// Prints credential.
-	printGatewayCommonMsg(apiEndPoints, accessKey, secretKey)
+	printGatewayCommonMsg(strippedAPIEndpoints)
 
 	// Prints `mc` cli configuration message chooses
 	// first endpoint as default.
-	endPoint := apiEndPoints[0]
-
-	// Configure 'mc', following block prints platform specific information for minio client.
-	log.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
-	if runtime.GOOS == globalWindowsOSName {
-		mcMessage := fmt.Sprintf("$ mc.exe config host add my%s %s %s %s", backendType, endPoint, accessKey, secretKey)
-		log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-	} else {
-		mcMessage := fmt.Sprintf("$ mc config host add my%s %s %s %s", backendType, endPoint, accessKey, secretKey)
-		log.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-	}
+	printCLIAccessMsg(strippedAPIEndpoints[0], fmt.Sprintf("my%s", backendType))
 
 	// Prints documentation message.
 	printObjectAPIMsg()
@@ -52,10 +43,16 @@ func printGatewayStartupMessage(apiEndPoints []string, accessKey, secretKey, bac
 }
 
 // Prints common server startup message. Prints credential, region and browser access.
-func printGatewayCommonMsg(apiEndpoints []string, accessKey, secretKey string) {
+func printGatewayCommonMsg(apiEndpoints []string) {
+	// Get saved credentials.
+	cred := serverConfig.GetCredential()
+
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 	// Colorize the message and print.
 	log.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
-	log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", accessKey)))
-	log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", secretKey)))
+	log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
+	log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+
+	log.Println(colorBlue("\nBrowser Access:"))
+	log.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
 }

cmd/gateway-startup-msg_test.go

@@ -20,12 +20,24 @@ import "testing"
 
 // Test printing Gateway common message.
 func TestPrintGatewayCommonMessage(t *testing.T) {
-	apiEndpoints := []string{"127.0.0.1:9000"}
-	printGatewayCommonMsg(apiEndpoints, "abcd1", "abcd123")
+	root, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeAll(root)
+
+	apiEndpoints := []string{"http://127.0.0.1:9000"}
+	printGatewayCommonMsg(apiEndpoints)
 }
 
 // Test print gateway startup message.
 func TestPrintGatewayStartupMessage(t *testing.T) {
-	apiEndpoints := []string{"127.0.0.1:9000"}
-	printGatewayStartupMessage(apiEndpoints, "abcd1", "abcd123", "azure")
+	root, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeAll(root)
+
+	apiEndpoints := []string{"http://127.0.0.1:9000"}
+	printGatewayStartupMessage(apiEndpoints, "azure")
 }

cmd/globals.go

@@ -17,20 +17,32 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"crypto/x509"
+	"os"
 	"runtime"
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/fatih/color"
+	miniohttp "github.com/minio/minio/pkg/http"
 )
 
 // minio configuration related constants.
 const (
 	globalMinioCertExpireWarnDays = time.Hour * 24 * 30 // 30 days.
 
-	globalMinioDefaultRegion       = "us-east-1"
-	globalMinioDefaultOwnerID      = "minio"
+	globalMinioDefaultRegion = ""
+	// This is a sha256 output of ``arn:aws:iam::minio:user/admin``,
+	// this is kept in present form to be compatible with S3 owner ID
+	// requirements -
+	//
+	// ```
+	//    The canonical user ID is the Amazon S3–only concept.
+	//    It is 64-character obfuscated version of the account ID.
+	// ```
+	// http://docs.aws.amazon.com/AmazonS3/latest/dev/example-walkthroughs-managing-access-example4.html
+	globalMinioDefaultOwnerID      = "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4"
 	globalMinioDefaultStorageClass = "STANDARD"
 	globalWindowsOSName            = "windows"
 	globalNetBSDOSName             = "netbsd"
@@ -40,6 +52,7 @@ const (
 	globalMinioModeDistXL          = "mode-server-distributed-xl"
 	globalMinioModeGatewayAzure    = "mode-gateway-azure"
 	globalMinioModeGatewayS3       = "mode-gateway-s3"
+	globalMinioModeGatewayGCS      = "mode-gateway-gcs"
 	// Add new global values here.
 )
 
@@ -64,6 +77,7 @@ var (
 
 	// This flag is set to 'true' by default
 	globalIsBrowserEnabled = true
+
 	// This flag is set to 'true' when MINIO_BROWSER env is set.
 	globalIsEnvBrowser = false
 
@@ -72,6 +86,7 @@ var (
 
 	// This flag is set to 'true' wen MINIO_REGION env is set.
 	globalIsEnvRegion = false
+
 	// This flag is set to 'us-east-1' by default
 	globalServerRegion = globalMinioDefaultRegion
 
@@ -94,6 +109,12 @@ var (
 	// IsSSL indicates if the server is configured with SSL.
 	globalIsSSL bool
 
+	globalTLSCertificate *tls.Certificate
+
+	globalHTTPServer        *miniohttp.Server
+	globalHTTPServerErrorCh = make(chan error)
+	globalOSSignalCh        = make(chan os.Signal, 1)
+
 	// List of admin peers.
 	globalAdminPeers = adminPeers{}
 
@@ -125,6 +146,27 @@ var (
 
 // global colors.
 var (
-	colorBold = color.New(color.Bold).SprintFunc()
-	colorBlue = color.New(color.FgBlue).SprintfFunc()
+	colorBold   = color.New(color.Bold).SprintFunc()
+	colorBlue   = color.New(color.FgBlue).SprintfFunc()
+	colorYellow = color.New(color.FgYellow).SprintfFunc()
 )
+
+// Returns minio global information, as a key value map.
+// returned list of global values is not an exhaustive
+// list. Feel free to add new relevant fields.
+func getGlobalInfo() (globalInfo map[string]interface{}) {
+	globalInfo = map[string]interface{}{
+		"isDistXL":         globalIsDistXL,
+		"isXL":             globalIsXL,
+		"isBrowserEnabled": globalIsBrowserEnabled,
+		"isEnvBrowser":     globalIsEnvBrowser,
+		"isEnvCreds":       globalIsEnvCreds,
+		"isEnvRegion":      globalIsEnvRegion,
+		"isSSL":            globalIsSSL,
+		"serverRegion":     globalServerRegion,
+		"serverUserAgent":  globalServerUserAgent,
+		// Add more relevant global settings here.
+	}
+
+	return globalInfo
+}

cmd/handler-utils.go

@@ -38,7 +38,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	} // else for both err as nil or io.EOF
 	location = locationConstraint.Location
 	if location == "" {
-		location = globalMinioDefaultRegion
+		location = serverConfig.GetRegion()
 	}
 	return location, ErrNone
 }
@@ -46,7 +46,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 // Validates input location is same as configured region
 // of Minio server.
 func isValidLocation(location string) bool {
-	return serverConfig.GetRegion() == location
+	return serverConfig.GetRegion() == "" || serverConfig.GetRegion() == location
 }
 
 // Supported headers that needs to be extracted.
@@ -98,9 +98,9 @@ func path2BucketAndObject(path string) (bucket, object string) {
 }
 
 // extractMetadataFromHeader extracts metadata from HTTP header.
-func extractMetadataFromHeader(header http.Header) map[string]string {
+func extractMetadataFromHeader(header http.Header) (map[string]string, error) {
 	if header == nil {
-		return nil
+		return nil, traceError(errInvalidArgument)
 	}
 	metadata := make(map[string]string)
 	// Save standard supported headers.
@@ -116,16 +116,17 @@ func extractMetadataFromHeader(header http.Header) map[string]string {
 	}
 	// Go through all other headers for any additional headers that needs to be saved.
 	for key := range header {
-		cKey := http.CanonicalHeaderKey(key)
-		if strings.HasPrefix(cKey, "X-Amz-Meta-") {
-			metadata[cKey] = header.Get(key)
-		} else if strings.HasPrefix(key, "X-Minio-Meta-") {
-			metadata[cKey] = header.Get(key)
+		if key != http.CanonicalHeaderKey(key) {
+			return nil, traceError(errInvalidArgument)
+		}
+		if strings.HasPrefix(key, "X-Amz-Meta-") {
+			metadata[key] = header.Get(key)
+		}
+		if strings.HasPrefix(key, "X-Minio-Meta-") {
+			metadata[key] = header.Get(key)
 		}
 	}
-
-	// Success.
-	return metadata
+	return metadata, nil
 }
 
 // The Query string for the redirect URL the client is
@@ -134,7 +135,7 @@ func getRedirectPostRawQuery(objInfo ObjectInfo) string {
 	redirectValues := make(url.Values)
 	redirectValues.Set("bucket", objInfo.Bucket)
 	redirectValues.Set("key", objInfo.Name)
-	redirectValues.Set("etag", "\""+objInfo.MD5Sum+"\"")
+	redirectValues.Set("etag", "\""+objInfo.ETag+"\"")
 	return redirectValues.Encode()
 }
 
@@ -168,11 +169,6 @@ func trimAwsChunkedContentEncoding(contentEnc string) (trimmedContentEnc string)
 	return strings.Join(newEncs, ",")
 }
 
-// extractMetadataFromForm extracts metadata from Post Form.
-func extractMetadataFromForm(formValues http.Header) map[string]string {
-	return extractMetadataFromHeader(formValues)
-}
-
 // Validate form field size for s3 specification requirement.
 func validateFormFieldSize(formValues http.Header) error {
 	// Iterate over form values

cmd/handler-utils_test.go

@@ -123,8 +123,9 @@ func TestValidateFormFieldSize(t *testing.T) {
 // Tests validate metadata extraction from http headers.
 func TestExtractMetadataHeaders(t *testing.T) {
 	testCases := []struct {
-		header   http.Header
-		metadata map[string]string
+		header     http.Header
+		metadata   map[string]string
+		shouldFail bool
 	}{
 		// Validate if there a known 'content-type'.
 		{
@@ -134,15 +135,17 @@ func TestExtractMetadataHeaders(t *testing.T) {
 			metadata: map[string]string{
 				"content-type": "image/png",
 			},
+			shouldFail: false,
 		},
 		// Validate if there are no keys to extract.
 		{
 			header: http.Header{
-				"test-1": []string{"123"},
+				"Test-1": []string{"123"},
 			},
-			metadata: map[string]string{},
+			metadata:   map[string]string{},
+			shouldFail: false,
 		},
-		// Validate if there are no keys to extract.
+		// Validate that there are all headers extracted
 		{
 			header: http.Header{
 				"X-Amz-Meta-Appid":   []string{"amz-meta"},
@@ -150,19 +153,38 @@ func TestExtractMetadataHeaders(t *testing.T) {
 			},
 			metadata: map[string]string{
 				"X-Amz-Meta-Appid":   "amz-meta",
-				"X-Minio-Meta-Appid": "minio-meta"},
+				"X-Minio-Meta-Appid": "minio-meta",
+			},
+			shouldFail: false,
+		},
+		// Fail if header key is not in canonicalized form
+		{
+			header: http.Header{
+				"x-amz-meta-appid": []string{"amz-meta"},
+			},
+			metadata: map[string]string{
+				"X-Amz-Meta-Appid": "amz-meta",
+			},
+			shouldFail: true,
 		},
 		// Empty header input returns empty metadata.
 		{
-			header:   nil,
-			metadata: nil,
+			header:     nil,
+			metadata:   nil,
+			shouldFail: true,
 		},
 	}
 
 	// Validate if the extracting headers.
 	for i, testCase := range testCases {
-		metadata := extractMetadataFromHeader(testCase.header)
-		if !reflect.DeepEqual(metadata, testCase.metadata) {
+		metadata, err := extractMetadataFromHeader(testCase.header)
+		if err != nil && !testCase.shouldFail {
+			t.Fatalf("Test %d failed to extract metadata: %v", i+1, err)
+		}
+		if err == nil && testCase.shouldFail {
+			t.Fatalf("Test %d should fail, but it passed", i+1)
+		}
+		if err == nil && !reflect.DeepEqual(metadata, testCase.metadata) {
 			t.Fatalf("Test %d failed: Expected \"%#v\", got \"%#v\"", i+1, testCase.metadata, metadata)
 		}
 	}

cmd/jwt_test.go

@@ -44,7 +44,7 @@ func testAuthenticate(authType string, t *testing.T) {
 		// Secret key too small.
 		{"myuser", "pass", errInvalidSecretKeyLength},
 		// Secret key too long.
-		{"myuser", "pass1234567890123456789012345678901234567", errInvalidSecretKeyLength},
+		{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", errInvalidSecretKeyLength},
 		// Authentication error.
 		{"myuser", "mypassword", errInvalidAccessKeyID},
 		// Authentication error.

cmd/lock-rpc-server-common.go

@@ -22,7 +22,7 @@ import (
 )
 
 // Similar to removeEntry but only removes an entry only if the lock entry exists in map.
-func (l *lockServer) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
+func (l *localLocker) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
 	// Check if entry is still in map (could have been removed altogether by 'concurrent' (R)Unlock of last entry)
 	if lri, ok := l.lockMap[nlrip.name]; ok {
 		if !l.removeEntry(nlrip.name, nlrip.lri.uid, &lri) {
@@ -38,7 +38,7 @@ func (l *lockServer) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
 
 // removeEntry either, based on the uid of the lock message, removes a single entry from the
 // lockRequesterInfo array or the whole array from the map (in case of a write lock or last read lock)
-func (l *lockServer) removeEntry(name, uid string, lri *[]lockRequesterInfo) bool {
+func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bool {
 	// Find correct entry to remove based on uid.
 	for index, entry := range *lri {
 		if entry.uid == uid {

cmd/lock-rpc-server-common_test.go

@@ -38,9 +38,9 @@ func TestLockRpcServerRemoveEntryIfExists(t *testing.T) {
 	nlrip := nameLockRequesterInfoPair{name: "name", lri: lri}
 
 	// first test by simulating item has already been deleted
-	locker.removeEntryIfExists(nlrip)
+	locker.ll.removeEntryIfExists(nlrip)
 	{
-		gotLri, _ := locker.lockMap["name"]
+		gotLri, _ := locker.ll.lockMap["name"]
 		expectedLri := []lockRequesterInfo(nil)
 		if !reflect.DeepEqual(expectedLri, gotLri) {
 			t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
@@ -48,10 +48,10 @@ func TestLockRpcServerRemoveEntryIfExists(t *testing.T) {
 	}
 
 	// then test normal deletion
-	locker.lockMap["name"] = []lockRequesterInfo{lri} // add item
-	locker.removeEntryIfExists(nlrip)
+	locker.ll.lockMap["name"] = []lockRequesterInfo{lri} // add item
+	locker.ll.removeEntryIfExists(nlrip)
 	{
-		gotLri, _ := locker.lockMap["name"]
+		gotLri, _ := locker.ll.lockMap["name"]
 		expectedLri := []lockRequesterInfo(nil)
 		if !reflect.DeepEqual(expectedLri, gotLri) {
 			t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
@@ -81,32 +81,32 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 		timeLastCheck:   UTCNow(),
 	}
 
-	locker.lockMap["name"] = []lockRequesterInfo{
+	locker.ll.lockMap["name"] = []lockRequesterInfo{
 		lockRequesterInfo1,
 		lockRequesterInfo2,
 	}
 
-	lri, _ := locker.lockMap["name"]
+	lri, _ := locker.ll.lockMap["name"]
 
 	// test unknown uid
-	if locker.removeEntry("name", "unknown-uid", &lri) {
+	if locker.ll.removeEntry("name", "unknown-uid", &lri) {
 		t.Errorf("Expected %#v, got %#v", false, true)
 	}
 
-	if !locker.removeEntry("name", "0123-4567", &lri) {
+	if !locker.ll.removeEntry("name", "0123-4567", &lri) {
 		t.Errorf("Expected %#v, got %#v", true, false)
 	} else {
-		gotLri, _ := locker.lockMap["name"]
+		gotLri, _ := locker.ll.lockMap["name"]
 		expectedLri := []lockRequesterInfo{lockRequesterInfo2}
 		if !reflect.DeepEqual(expectedLri, gotLri) {
 			t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
 		}
 	}
 
-	if !locker.removeEntry("name", "89ab-cdef", &lri) {
+	if !locker.ll.removeEntry("name", "89ab-cdef", &lri) {
 		t.Errorf("Expected %#v, got %#v", true, false)
 	} else {
-		gotLri, _ := locker.lockMap["name"]
+		gotLri, _ := locker.ll.lockMap["name"]
 		expectedLri := []lockRequesterInfo(nil)
 		if !reflect.DeepEqual(expectedLri, gotLri) {
 			t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)

cmd/lock-rpc-server.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"fmt"
 	"math/rand"
-	"net/rpc"
 	"path"
 	"sync"
 	"time"
@@ -60,13 +59,11 @@ func isWriteLock(lri []lockRequesterInfo) bool {
 // lockServer is type for RPC handlers
 type lockServer struct {
 	AuthRPCServer
-	serviceEndpoint string
-	mutex           sync.Mutex
-	lockMap         map[string][]lockRequesterInfo
+	ll localLocker
 }
 
 // Start lock maintenance from all lock servers.
-func startLockMaintainence(lockServers []*lockServer) {
+func startLockMaintenance(lockServers []*lockServer) {
 	for _, locker := range lockServers {
 		// Start loop for stale lock maintenance
 		go func(lk *lockServer) {
@@ -91,163 +88,193 @@ func startLockMaintainence(lockServers []*lockServer) {
 
 // Register distributed NS lock handlers.
 func registerDistNSLockRouter(mux *router.Router, endpoints EndpointList) error {
-	// Initialize a new set of lock servers.
-	lockServers := newLockServers(endpoints)
-
 	// Start lock maintenance from all lock servers.
-	startLockMaintainence(lockServers)
+	startLockMaintenance(globalLockServers)
 
 	// Register initialized lock servers to their respective rpc endpoints.
-	return registerStorageLockers(mux, lockServers)
-}
-
-// Create one lock server for every local storage rpc server.
-func newLockServers(endpoints EndpointList) (lockServers []*lockServer) {
-	for _, endpoint := range endpoints {
-		// Initialize new lock server for each local node.
-		if endpoint.IsLocal {
-			lockServers = append(lockServers, &lockServer{
-				serviceEndpoint: endpoint.Path,
-				mutex:           sync.Mutex{},
-				lockMap:         make(map[string][]lockRequesterInfo),
-			})
-		}
-	}
-
-	return lockServers
+	return registerStorageLockers(mux, globalLockServers)
 }
 
 // registerStorageLockers - register locker rpc handlers for net/rpc library clients
 func registerStorageLockers(mux *router.Router, lockServers []*lockServer) error {
 	for _, lockServer := range lockServers {
-		lockRPCServer := rpc.NewServer()
+		lockRPCServer := newRPCServer()
 		if err := lockRPCServer.RegisterName(lockServiceName, lockServer); err != nil {
 			return traceError(err)
 		}
 		lockRouter := mux.PathPrefix(minioReservedBucketPath).Subrouter()
-		lockRouter.Path(path.Join(lockServicePath, lockServer.serviceEndpoint)).Handler(lockRPCServer)
+		lockRouter.Path(path.Join(lockServicePath, lockServer.ll.serviceEndpoint)).Handler(lockRPCServer)
 	}
 	return nil
 }
 
-///  Distributed lock handlers
+// localLocker implements Dsync.NetLocker
+type localLocker struct {
+	mutex           sync.Mutex
+	serviceEndpoint string
+	serverAddr      string
+	lockMap         map[string][]lockRequesterInfo
+}
 
-// Lock - rpc handler for (single) write lock operation.
-func (l *lockServer) Lock(args *LockArgs, reply *bool) error {
+func (l *localLocker) ServerAddr() string {
+	return l.serverAddr
+}
+
+func (l *localLocker) ServiceEndpoint() string {
+	return l.serviceEndpoint
+}
+
+func (l *localLocker) Lock(args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
-	_, *reply = l.lockMap[args.LockArgs.Resource]
-	if !*reply { // No locks held on the given name, so claim write lock
-		l.lockMap[args.LockArgs.Resource] = []lockRequesterInfo{
+	_, isLockTaken := l.lockMap[args.Resource]
+	if !isLockTaken { // No locks held on the given name, so claim write lock
+		l.lockMap[args.Resource] = []lockRequesterInfo{
 			{
 				writer:          true,
-				node:            args.LockArgs.ServerAddr,
-				serviceEndpoint: args.LockArgs.ServiceEndpoint,
-				uid:             args.LockArgs.UID,
+				node:            args.ServerAddr,
+				serviceEndpoint: args.ServiceEndpoint,
+				uid:             args.UID,
 				timestamp:       UTCNow(),
 				timeLastCheck:   UTCNow(),
 			},
 		}
 	}
-	*reply = !*reply // Negate *reply to return true when lock is granted or false otherwise
-	return nil
+	// return reply=true if lock was granted.
+	return !isLockTaken, nil
 }
 
-// Unlock - rpc handler for (single) write unlock operation.
-func (l *lockServer) Unlock(args *LockArgs, reply *bool) error {
+func (l *localLocker) Unlock(args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
 	var lri []lockRequesterInfo
-	if lri, *reply = l.lockMap[args.LockArgs.Resource]; !*reply { // No lock is held on the given name
-		return fmt.Errorf("Unlock attempted on an unlocked entity: %s", args.LockArgs.Resource)
+	if lri, reply = l.lockMap[args.Resource]; !reply {
+		// No lock is held on the given name
+		return reply, fmt.Errorf("Unlock attempted on an unlocked entity: %s", args.Resource)
 	}
-	if *reply = isWriteLock(lri); !*reply { // Unless it is a write lock
-		return fmt.Errorf("Unlock attempted on a read locked entity: %s (%d read locks active)", args.LockArgs.Resource, len(lri))
+	if reply = isWriteLock(lri); !reply {
+		// Unless it is a write lock
+		return reply, fmt.Errorf("Unlock attempted on a read locked entity: %s (%d read locks active)", args.Resource, len(lri))
 	}
-	if !l.removeEntry(args.LockArgs.Resource, args.LockArgs.UID, &lri) {
-		return fmt.Errorf("Unlock unable to find corresponding lock for uid: %s", args.LockArgs.UID)
+	if !l.removeEntry(args.Resource, args.UID, &lri) {
+		return false, fmt.Errorf("Unlock unable to find corresponding lock for uid: %s", args.UID)
 	}
-	return nil
+	return true, nil
+
 }
 
-// RLock - rpc handler for read lock operation.
-func (l *lockServer) RLock(args *LockArgs, reply *bool) error {
+func (l *localLocker) RLock(args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
-	if err := args.IsAuthenticated(); err != nil {
-		return err
-	}
 	lrInfo := lockRequesterInfo{
 		writer:          false,
-		node:            args.LockArgs.ServerAddr,
-		serviceEndpoint: args.LockArgs.ServiceEndpoint,
-		uid:             args.LockArgs.UID,
+		node:            args.ServerAddr,
+		serviceEndpoint: args.ServiceEndpoint,
+		uid:             args.UID,
 		timestamp:       UTCNow(),
 		timeLastCheck:   UTCNow(),
 	}
-	if lri, ok := l.lockMap[args.LockArgs.Resource]; ok {
-		if *reply = !isWriteLock(lri); *reply { // Unless there is a write lock
-			l.lockMap[args.LockArgs.Resource] = append(l.lockMap[args.LockArgs.Resource], lrInfo)
+	if lri, ok := l.lockMap[args.Resource]; ok {
+		if reply = !isWriteLock(lri); reply {
+			// Unless there is a write lock
+			l.lockMap[args.Resource] = append(l.lockMap[args.Resource], lrInfo)
 		}
-	} else { // No locks held on the given name, so claim (first) read lock
-		l.lockMap[args.LockArgs.Resource] = []lockRequesterInfo{lrInfo}
-		*reply = true
+	} else {
+		// No locks held on the given name, so claim (first) read lock
+		l.lockMap[args.Resource] = []lockRequesterInfo{lrInfo}
+		reply = true
 	}
-	return nil
+	return reply, nil
+}
+
+func (l *localLocker) RUnlock(args dsync.LockArgs) (reply bool, err error) {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	var lri []lockRequesterInfo
+	if lri, reply = l.lockMap[args.Resource]; !reply {
+		// No lock is held on the given name
+		return reply, fmt.Errorf("RUnlock attempted on an unlocked entity: %s", args.Resource)
+	}
+	if reply = !isWriteLock(lri); !reply {
+		// A write-lock is held, cannot release a read lock
+		return reply, fmt.Errorf("RUnlock attempted on a write locked entity: %s", args.Resource)
+	}
+	if !l.removeEntry(args.Resource, args.UID, &lri) {
+		return false, fmt.Errorf("RUnlock unable to find corresponding read lock for uid: %s", args.UID)
+	}
+	return reply, nil
+}
+
+func (l *localLocker) ForceUnlock(args dsync.LockArgs) (reply bool, err error) {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	if len(args.UID) != 0 {
+		return false, fmt.Errorf("ForceUnlock called with non-empty UID: %s", args.UID)
+	}
+	if _, ok := l.lockMap[args.Resource]; ok {
+		// Only clear lock when it is taken
+		// Remove the lock (irrespective of write or read lock)
+		delete(l.lockMap, args.Resource)
+	}
+	return true, nil
+}
+
+///  Distributed lock handlers
+
+// Lock - rpc handler for (single) write lock operation.
+func (l *lockServer) Lock(args *LockArgs, reply *bool) (err error) {
+	if err = args.IsAuthenticated(); err != nil {
+		return err
+	}
+	*reply, err = l.ll.Lock(args.LockArgs)
+	return err
+}
+
+// Unlock - rpc handler for (single) write unlock operation.
+func (l *lockServer) Unlock(args *LockArgs, reply *bool) (err error) {
+	if err = args.IsAuthenticated(); err != nil {
+		return err
+	}
+	*reply, err = l.ll.Unlock(args.LockArgs)
+	return err
+}
+
+// RLock - rpc handler for read lock operation.
+func (l *lockServer) RLock(args *LockArgs, reply *bool) (err error) {
+	if err = args.IsAuthenticated(); err != nil {
+		return err
+	}
+	*reply, err = l.ll.RLock(args.LockArgs)
+	return err
 }
 
 // RUnlock - rpc handler for read unlock operation.
-func (l *lockServer) RUnlock(args *LockArgs, reply *bool) error {
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
-	if err := args.IsAuthenticated(); err != nil {
+func (l *lockServer) RUnlock(args *LockArgs, reply *bool) (err error) {
+	if err = args.IsAuthenticated(); err != nil {
 		return err
 	}
-	var lri []lockRequesterInfo
-	if lri, *reply = l.lockMap[args.LockArgs.Resource]; !*reply { // No lock is held on the given name
-		return fmt.Errorf("RUnlock attempted on an unlocked entity: %s", args.LockArgs.Resource)
-	}
-	if *reply = !isWriteLock(lri); !*reply { // A write-lock is held, cannot release a read lock
-		return fmt.Errorf("RUnlock attempted on a write locked entity: %s", args.LockArgs.Resource)
-	}
-	if !l.removeEntry(args.LockArgs.Resource, args.LockArgs.UID, &lri) {
-		return fmt.Errorf("RUnlock unable to find corresponding read lock for uid: %s", args.LockArgs.UID)
-	}
-	return nil
+	*reply, err = l.ll.RUnlock(args.LockArgs)
+	return err
 }
 
 // ForceUnlock - rpc handler for force unlock operation.
-func (l *lockServer) ForceUnlock(args *LockArgs, reply *bool) error {
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
-	if err := args.IsAuthenticated(); err != nil {
+func (l *lockServer) ForceUnlock(args *LockArgs, reply *bool) (err error) {
+	if err = args.IsAuthenticated(); err != nil {
 		return err
 	}
-	if len(args.LockArgs.UID) != 0 {
-		return fmt.Errorf("ForceUnlock called with non-empty UID: %s", args.LockArgs.UID)
-	}
-	if _, ok := l.lockMap[args.LockArgs.Resource]; ok { // Only clear lock when set
-		delete(l.lockMap, args.LockArgs.Resource) // Remove the lock (irrespective of write or read lock)
-	}
-	*reply = true
-	return nil
+	*reply, err = l.ll.ForceUnlock(args.LockArgs)
+	return err
 }
 
 // Expired - rpc handler for expired lock status.
 func (l *lockServer) Expired(args *LockArgs, reply *bool) error {
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
 	if err := args.IsAuthenticated(); err != nil {
 		return err
 	}
+	l.ll.mutex.Lock()
+	defer l.ll.mutex.Unlock()
 	// Lock found, proceed to verify if belongs to given uid.
-	if lri, ok := l.lockMap[args.LockArgs.Resource]; ok {
+	if lri, ok := l.ll.lockMap[args.LockArgs.Resource]; ok {
 		// Check whether uid is still active
 		for _, entry := range lri {
 			if entry.uid == args.LockArgs.UID {
@@ -277,10 +304,10 @@ type nameLockRequesterInfoPair struct {
 //
 // We will ignore the error, and we will retry later to get a resolve on this lock
 func (l *lockServer) lockMaintenance(interval time.Duration) {
-	l.mutex.Lock()
+	l.ll.mutex.Lock()
 	// Get list of long lived locks to check for staleness.
-	nlripLongLived := getLongLivedLocks(l.lockMap, interval)
-	l.mutex.Unlock()
+	nlripLongLived := getLongLivedLocks(l.ll.lockMap, interval)
+	l.ll.mutex.Unlock()
 
 	serverCred := serverConfig.GetCredential()
 	// Validate if long lived locks are indeed clean.
@@ -308,9 +335,9 @@ func (l *lockServer) lockMaintenance(interval time.Duration) {
 		if expired {
 			// The lock is no longer active at server that originated the lock
 			// So remove the lock from the map.
-			l.mutex.Lock()
-			l.removeEntryIfExists(nlrip) // Purge the stale entry if it exists.
-			l.mutex.Unlock()
+			l.ll.mutex.Lock()
+			l.ll.removeEntryIfExists(nlrip) // Purge the stale entry if it exists.
+			l.ll.mutex.Unlock()
 		}
 	}
 }

cmd/lock-rpc-server_test.go

@@ -49,10 +49,12 @@ func createLockTestServer(t *testing.T) (string, *lockServer, string) {
 	}
 
 	locker := &lockServer{
-		AuthRPCServer:   AuthRPCServer{},
-		serviceEndpoint: "rpc-path",
-		mutex:           sync.Mutex{},
-		lockMap:         make(map[string][]lockRequesterInfo),
+		AuthRPCServer: AuthRPCServer{},
+		ll: localLocker{
+			mutex:           sync.Mutex{},
+			serviceEndpoint: "rpc-path",
+			lockMap:         make(map[string][]lockRequesterInfo),
+		},
 	}
 	creds := serverConfig.GetCredential()
 	loginArgs := LoginRPCArgs{
@@ -93,7 +95,7 @@ func TestLockRpcServerLock(t *testing.T) {
 		if !result {
 			t.Errorf("Expected %#v, got %#v", true, result)
 		} else {
-			gotLri, _ := locker.lockMap["name"]
+			gotLri, _ := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo{
 				{
 					writer:          true,
@@ -163,7 +165,7 @@ func TestLockRpcServerUnlock(t *testing.T) {
 		if !result {
 			t.Errorf("Expected %#v, got %#v", true, result)
 		} else {
-			gotLri, _ := locker.lockMap["name"]
+			gotLri, _ := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo(nil)
 			if !testLockEquality(expectedLri, gotLri) {
 				t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
@@ -194,7 +196,7 @@ func TestLockRpcServerRLock(t *testing.T) {
 		if !result {
 			t.Errorf("Expected %#v, got %#v", true, result)
 		} else {
-			gotLri, _ := locker.lockMap["name"]
+			gotLri, _ := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo{
 				{
 					writer:          false,
@@ -281,7 +283,7 @@ func TestLockRpcServerRUnlock(t *testing.T) {
 		if !result {
 			t.Errorf("Expected %#v, got %#v", true, result)
 		} else {
-			gotLri, _ := locker.lockMap["name"]
+			gotLri, _ := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo{
 				{
 					writer:          false,
@@ -305,7 +307,7 @@ func TestLockRpcServerRUnlock(t *testing.T) {
 		if !result {
 			t.Errorf("Expected %#v, got %#v", true, result)
 		} else {
-			gotLri, _ := locker.lockMap["name"]
+			gotLri, _ := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo(nil)
 			if !testLockEquality(expectedLri, gotLri) {
 				t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
@@ -427,6 +429,12 @@ func TestLockServers(t *testing.T) {
 		return
 	}
 
+	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	defer removeAll(rootPath)
+
 	currentIsDistXL := globalIsDistXL
 	defer func() {
 		globalIsDistXL = currentIsDistXL
@@ -471,9 +479,13 @@ func TestLockServers(t *testing.T) {
 	// Validates lock server initialization.
 	for i, testCase := range testCases {
 		globalIsDistXL = testCase.isDistXL
-		lockServers := newLockServers(testCase.endpoints)
-		if len(lockServers) != testCase.totalLockServers {
-			t.Fatalf("Test %d: Expected total %d, got %d", i+1, testCase.totalLockServers, len(lockServers))
+		globalLockServers = nil
+		_, _ = newDsyncNodes(testCase.endpoints)
+		if err != nil {
+			t.Fatalf("Got unexpected error initializing lock servers: %v", err)
+		}
+		if len(globalLockServers) != testCase.totalLockServers {
+			t.Fatalf("Test %d: Expected total %d, got %d", i+1, testCase.totalLockServers, len(globalLockServers))
 		}
 	}
 }

cmd/main.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"os"
+	"path/filepath"
 	"sort"
 
 	"github.com/minio/cli"
@@ -59,7 +60,7 @@ VERSION:
   ` + Version +
 	`{{ "\n"}}`
 
-func newApp() *cli.App {
+func newApp(name string) *cli.App {
 	// Collection of minio commands currently supported are.
 	commands := []cli.Command{}
 
@@ -108,7 +109,7 @@ func newApp() *cli.App {
 	}
 
 	app := cli.NewApp()
-	app.Name = "Minio"
+	app.Name = name
 	app.Author = "Minio.io"
 	app.Version = Version
 	app.Usage = "Cloud Storage Server."
@@ -137,10 +138,11 @@ func newApp() *cli.App {
 
 // Main main for minio server.
 func Main(args []string) {
-	app := newApp()
+	// Set the minio app name.
+	appName := filepath.Base(args[0])
 
 	// Run the app - exit on error.
-	if err := app.Run(args); err != nil {
+	if err := newApp(appName).Run(args); err != nil {
 		os.Exit(1)
 	}
 }

cmd/namespace-lock.go

@@ -27,6 +27,9 @@ import (
 // Global name space lock.
 var globalNSMutex *nsLockMap
 
+// Global lock servers
+var globalLockServers []*lockServer
+
 // RWLocker - locker interface extends sync.Locker
 // to introduce RLock, RUnlock.
 type RWLocker interface {
@@ -36,27 +39,45 @@ type RWLocker interface {
 }
 
 // Initialize distributed locking only in case of distributed setup.
-// Returns if the setup is distributed or not on success.
-func initDsyncNodes() error {
+// Returns lock clients and the node index for the current server.
+func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int) {
 	cred := serverConfig.GetCredential()
-	// Initialize rpc lock client information only if this instance is a distributed setup.
-	clnts := make([]dsync.NetLocker, len(globalEndpoints))
-	myNode := -1
-	for index, endpoint := range globalEndpoints {
-		clnts[index] = newLockRPCClient(authConfig{
-			accessKey:       cred.AccessKey,
-			secretKey:       cred.SecretKey,
-			serverAddr:      endpoint.Host,
-			secureConn:      globalIsSSL,
-			serviceEndpoint: pathutil.Join(minioReservedBucketPath, lockServicePath, endpoint.Path),
-			serviceName:     lockServiceName,
-		})
-		if endpoint.IsLocal && myNode == -1 {
+	clnts = make([]dsync.NetLocker, len(endpoints))
+	myNode = -1
+	for index, endpoint := range endpoints {
+		if !endpoint.IsLocal {
+			// For a remote endpoints setup a lock RPC client.
+			clnts[index] = newLockRPCClient(authConfig{
+				accessKey:       cred.AccessKey,
+				secretKey:       cred.SecretKey,
+				serverAddr:      endpoint.Host,
+				secureConn:      globalIsSSL,
+				serviceEndpoint: pathutil.Join(minioReservedBucketPath, lockServicePath, endpoint.Path),
+				serviceName:     lockServiceName,
+			})
+			continue
+		}
+
+		// Local endpoint
+		if myNode == -1 {
 			myNode = index
 		}
+		// For a local endpoint, setup a local lock server to
+		// avoid network requests.
+		localLockServer := lockServer{
+			AuthRPCServer: AuthRPCServer{},
+			ll: localLocker{
+				mutex:           sync.Mutex{},
+				serviceEndpoint: endpoint.Path,
+				serverAddr:      endpoint.Host,
+				lockMap:         make(map[string][]lockRequesterInfo),
+			},
+		}
+		globalLockServers = append(globalLockServers, &localLockServer)
+		clnts[index] = &(localLockServer.ll)
 	}
 
-	return dsync.Init(clnts, myNode)
+	return clnts, myNode
 }
 
 // initNSLock - initialize name space lock map.
@@ -240,14 +261,12 @@ func (n *nsLockMap) ForceUnlock(volume, path string) {
 	if _, found := n.lockMap[param]; found {
 		// Remove lock from the map.
 		delete(n.lockMap, param)
-
-		// delete the lock state entry for given
-		// <volume, path> pair.
-		err := n.deleteLockInfoEntryForVolumePath(param)
-		if err != nil {
-			errorIf(err, "Failed to delete lock info entry")
-		}
 	}
+
+	// delete the lock state entry for given
+	// <volume, path> pair. Ignore error as there
+	// is no way to report it back
+	n.deleteLockInfoEntryForVolumePath(param)
 }
 
 // lockInstance - frontend/top-level interface for namespace locks.

cmd/naughty-disk_test.go

@@ -122,6 +122,15 @@ func (d *naughtyDisk) ReadFile(volume string, path string, offset int64, buf []b
 	return d.disk.ReadFile(volume, path, offset, buf)
 }
 
+func (d *naughtyDisk) ReadFileWithVerify(volume, path string, offset int64,
+	buf []byte, algo HashAlgo, expectedHash string) (n int64, err error) {
+
+	if err := d.calcError(); err != nil {
+		return 0, err
+	}
+	return d.disk.ReadFileWithVerify(volume, path, offset, buf, algo, expectedHash)
+}
+
 func (d *naughtyDisk) PrepareFile(volume, path string, length int64) error {
 	if err := d.calcError(); err != nil {
 		return err