Checking updates should send a proper user-agent.

Fixes #2799

Dockerfile

@@ -1,15 +1,17 @@
-FROM golang:1.6
+FROM golang:1.7-alpine
 
-RUN mkdir -p /go/src/app
 WORKDIR /go/src/app
 
 COPY . /go/src/app
-RUN go-wrapper download
-RUN go-wrapper install
-
-ENV ALLOW_CONTAINER_ROOT=1
-RUN mkdir -p /export/docker && cp /go/src/app/docs/Docker.md /export/docker/
+RUN \
+	apk add --no-cache git && \
+	go-wrapper download && \
+	go-wrapper install -ldflags "-X github.com/minio/minio/cmd.Version=2016-10-07T01:16:39Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2016-10-07T01-16-39Z -X github.com/minio/minio/cmd.CommitID=b06a514ce0cf10f1d4c8bbc7b5cdbd956c567ab9" && \
+	mkdir -p /export/docker && \
+	cp /go/src/app/docs/Docker.md /export/docker/ && \
+	rm -rf /go/pkg /go/src && \
+	apk del git
 
 EXPOSE 9000
-ENTRYPOINT ["go-wrapper", "run", "server"]
-CMD ["/export"]
+ENTRYPOINT ["go-wrapper", "run"]
+VOLUME ["/export"]

Makefile

@@ -71,19 +71,19 @@ verifiers: vet fmt lint cyclo spelling
 
 vet:
 	@echo "Running $@:"
-	@GO15VENDOREXPERIMENT=1 go tool vet -all *.go
+	@GO15VENDOREXPERIMENT=1 go tool vet -all ./cmd
 	@GO15VENDOREXPERIMENT=1 go tool vet -all ./pkg
-	@GO15VENDOREXPERIMENT=1 go tool vet -shadow=true *.go
+	@GO15VENDOREXPERIMENT=1 go tool vet -shadow=true ./cmd
 	@GO15VENDOREXPERIMENT=1 go tool vet -shadow=true ./pkg
 
 fmt:
 	@echo "Running $@:"
-	@GO15VENDOREXPERIMENT=1 gofmt -s -l *.go
+	@GO15VENDOREXPERIMENT=1 gofmt -s -l cmd
 	@GO15VENDOREXPERIMENT=1 gofmt -s -l pkg
 
 lint:
 	@echo "Running $@:"
-	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint *.go
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint github.com/minio/minio/cmd...
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint github.com/minio/minio/pkg...
 
 ineffassign:
@@ -92,7 +92,7 @@ ineffassign:
 
 cyclo:
 	@echo "Running $@:"
-	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 *.go
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 cmd
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 pkg
 
 build: getdeps verifiers $(UI_ASSETS)
@@ -101,12 +101,12 @@ deadcode:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
 
 spelling:
-	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell -error *.go
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell -error cmd/**/*
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell -error pkg/**/*
 
 test: build
 	@echo "Running all minio testing:"
-	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
+	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/cmd...
 	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
 
 coverage: build
@@ -139,8 +139,6 @@ experimental: verifiers
 
 clean:
 	@echo "Cleaning up all the generated files:"
-	@rm -fv minio minio.test cover.out
 	@find . -name '*.test' | xargs rm -fv
-	@rm -rf isa-l
 	@rm -rf build
 	@rm -rf release

README.md

@@ -1,9 +1,8 @@
-# Minio Quickstart Guide [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+# Minio Quickstart Guide [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
 
 Minio is an object storage server released under Apache License v2.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
 
 ##  1. Download
-
 Minio server is light enough to be bundled with the application stack, similar to NodeJS, Redis and MySQL.
 
 | Platform| Architecture | URL|
@@ -16,169 +15,88 @@ Minio server is light enough to be bundled with the application stack, similar t
 ||32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe|
 |FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
 
-### Install from Source
+### Install from Homebrew
+Install minio packages using [Homebrew](http://brew.sh/)
+ 
+```sh
+$ brew install minio
+$ minio --help
+```
 
+### Install from Source
 Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
 
 
 ```sh
-
-$ go get -d github.com/minio/minio
-$ cd $GOPATH/src/github.com/minio/minio
-$ make
-
+$ go get -u github.com/minio/minio
 ```
 
 ## 2. Run Minio Server
+In the examples below, Minio serves the contents of the ``Photos`` directory as an object store.
 
+### Docker Container
+
+```sh
+$ docker pull minio/minio
+$ docker run -p 9000:9000 minio/minio server /export
+```
+
+Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
 ### GNU/Linux
 
- ```sh
-
+```sh
 $ chmod +x minio
 $ ./minio --help
 $ ./minio server ~/Photos
-
-Endpoint:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-AccessKey: USWUXHGYZQYFYFFIT3RE
-SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-Region:    us-east-1
-
-Browser Access:
-   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-
-Command-line Access: https://docs.minio.io/docs/minio-client-quickstart-guide
-   $ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-
-Object API (Amazon S3 compatible):
-   Go:         https://docs.minio.io/docs/golang-client-quickstart-guide
-   Java:       https://docs.minio.io/docs/java-client-quickstart-guide
-   Python:     https://docs.minio.io/docs/python-client-quickstart-guide
-   JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide
-
+...
 ```
 
 ### OS X
 
-
- ```sh
-
+```sh
 $ chmod 755 minio
 $ ./minio --help
 $ ./minio server ~/Photos
-
-Endpoint:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-AccessKey: USWUXHGYZQYFYFFIT3RE
-SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-Region:    us-east-1
-
-Browser Access:
-   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-
-Command-line Access: https://docs.minio.io/docs/minio-client-quickstart-guide
-   $ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-
-Object API (Amazon S3 compatible):
-   Go:         https://docs.minio.io/docs/golang-client-quickstart-guide
-   Java:       https://docs.minio.io/docs/java-client-quickstart-guide
-   Python:     https://docs.minio.io/docs/python-client-quickstart-guide
-   JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide
-
+...
 ```
 
 ### Microsoft Windows
 
 ```sh
-
 C:\Users\Username\Downloads> minio.exe --help
 C:\Users\Username\Downloads> minio.exe server D:\Photos
-
-Endpoint:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-AccessKey: USWUXHGYZQYFYFFIT3RE
-SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-Region:    us-east-1
-
-Browser Access:
-   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-
-Command-line Access: https://docs.minio.io/docs/minio-client-quickstart-guide
-   $ mc.exe config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-
-Object API (Amazon S3 compatible):
-   Go:         https://docs.minio.io/docs/golang-client-quickstart-guide
-   Java:       https://docs.minio.io/docs/java-client-quickstart-guide
-   Python:     https://docs.minio.io/docs/python-client-quickstart-guide
-   JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide
-
-
-```
-
-### Docker Container
-
-```sh
-
-$ docker pull minio/minio
-$ docker run -p 9000:9000 minio/minio
-
+...
 ```
 
 ### FreeBSD
 
 ```sh
-
 $ chmod 755 minio
 $ ./minio --help
 $ ./minio server ~/Photos
-
-Endpoint:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-AccessKey: USWUXHGYZQYFYFFIT3RE
-SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-Region:    us-east-1
-
-Browser Access:
-   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
-
-Command-line Access: https://docs.minio.io/docs/minio-client-quickstart-guide
-   $ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
-
-Object API (Amazon S3 compatible):
-   Go:         https://docs.minio.io/docs/golang-client-quickstart-guide
-   Java:       https://docs.minio.io/docs/java-client-quickstart-guide
-   Python:     https://docs.minio.io/docs/python-client-quickstart-guide
-   JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide
-
-
+...
 ```
+Please visit official zfs FreeBSD guide for more details [here](https://www.freebsd.org/doc/handbook/zfs-quickstart.html)
 
 ## 3. Test Minio Server using Minio Browser
-
 Open a web browser and navigate to http://127.0.0.1:9000 to view your buckets on minio server.
 
 ![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true)
 
-
 ## 4. Test Minio Server using `mc`
-
-
 Install mc from [here](https://docs.minio.io/docs/minio-client-quickstart-guide). Use `mc ls` command to list all the buckets on your minio server.
 
 ```sh
-
+$ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+$ mc mb myminio/mybucket
 $ mc ls myminio/
-[2015-08-05 08:13:22 IST]     0B andoria/
-[2015-08-05 06:14:26 IST]     0B deflector/
-[2015-08-05 08:13:11 IST]     0B ferenginar/
-[2016-03-08 14:56:35 IST]     0B jarjarbing/
-[2016-01-20 16:07:41 IST]     0B my.minio.io/
-
+[2015-08-05 08:13:11 IST]     0B mybucket/
 ```
 
 For more examples please navigate to [Minio Client Complete Guide](https://docs.minio.io/docs/minio-client-complete-guide).
 
-
 ## 5. Explore Further
-
 - [Minio Erasure Code QuickStart Guide](https://docs.minio.io/docs/minio-erasure-code-quickstart-guide)
 - [Minio Docker Quickstart Guide](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 - [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)
@@ -186,6 +104,5 @@ For more examples please navigate to [Minio Client Complete Guide](https://docs.
 - [Use `s3cmd` with Minio Server](https://docs.minio.io/docs/s3cmd-with-minio)
 - [Use `minio-go` SDK with Minio Server](https://docs.minio.io/docs/golang-client-quickstart-guide)
 
-
 ## 6. Contribute to Minio Project
 Please follow Minio [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)

appveyor.yml

@@ -9,7 +9,7 @@ platform: x64
 
 clone_folder: c:\gopath\src\github.com\minio\minio
 
-# environment variables
+# Environment variables
 environment:
   GOPATH: c:\gopath
   GO15VENDOREXPERIMENT: 1
@@ -19,21 +19,36 @@ install:
   - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
   - go version
   - go env
+  - python --version
 
-# to run your custom scripts instead of automatic MSBuild
+# To run your custom scripts instead of automatic MSBuild
 build_script:
-  - go test -race .
-  - go test -race github.com/minio/minio/pkg...
-  - go test -coverprofile=coverage.txt -covermode=atomic
+  # Compile
+  - appveyor AddCompilationMessage "Starting Compile"
+  - cd c:\gopath\src\github.com\minio\minio
   - go run buildscripts/gen-ldflags.go > temp.txt
   - set /p BUILD_LDFLAGS=<temp.txt
   - go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
+  - appveyor AddCompilationMessage "Compile Success"
+
+# To run your custom scripts instead of automatic tests
+test_script:
+  # Unit tests
+  - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
+  - mkdir build\coverage
+  - go test -race github.com/minio/minio/cmd...
+  - go test -race github.com/minio/minio/pkg...
+  - go test -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
+  - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
 
 after_test:
-  - bash <(curl -s https://codecov.io/bash)
-
-# to disable automatic tests
-test: off
+  - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
+  - ps: Push-AppveyorArtifact build\coverage\coverage.txt
+  - ps: Push-AppveyorArtifact build\coverage\coverage.html
+  # Upload coverage report.
+  - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
+  - pip install codecov
+  - codecov -X gcov -f "build\coverage\coverage.txt"
 
 # to disable deployment
 deploy: off

buildscripts/gen-ldflags.go

@@ -28,11 +28,11 @@ import (
 
 func genLDFlags(version string) string {
 	var ldflagsStr string
-	ldflagsStr = "-X main.minioVersion=" + version
-	ldflagsStr += " -X main.minioReleaseTag=" + releaseTag(version)
-	ldflagsStr += " -X main.minioCommitID=" + commitID()
-	ldflagsStr += " -X main.minioShortCommitID=" + commitID()[:12]
-	ldflagsStr += " -X main.minioGOPATH=" + os.Getenv("GOPATH")
+	ldflagsStr = "-X github.com/minio/minio/cmd.Version=" + version
+	ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag(version)
+	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
+	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
+	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
 	return ldflagsStr
 }
 

cmd/access-key.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"crypto/rand"

cmd/api-datatypes.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"encoding/xml"

cmd/api-errors.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"encoding/xml"
@@ -103,6 +103,7 @@ const (
 	ErrNegativeExpires
 	ErrAuthHeaderEmpty
 	ErrExpiredPresignRequest
+	ErrRequestNotReadyYet
 	ErrUnsignedHeaders
 	ErrMissingDateHeader
 	ErrInvalidQuerySignatureAlgo
@@ -118,7 +119,8 @@ const (
 	ErrFilterNameInvalid
 	ErrFilterNamePrefix
 	ErrFilterNameSuffix
-	ErrFilterPrefixValueInvalid
+	ErrFilterValueInvalid
+	ErrOverlappingConfigs
 
 	// S3 extended errors.
 	ErrContentSHA256Mismatch
@@ -262,7 +264,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrNoSuchBucketPolicy: {
 		Code:           "NoSuchBucketPolicy",
-		Description:    "The specified bucket does not have a bucket policy.",
+		Description:    "The bucket policy does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrNoSuchKey: {
@@ -327,7 +329,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrBucketNotEmpty: {
 		Code:           "BucketNotEmpty",
-		Description:    "The bucket you tried to delete is not empty.",
+		Description:    "The bucket you tried to delete is not empty",
 		HTTPStatusCode: http.StatusConflict,
 	},
 	ErrAllAccessDisabled: {
@@ -445,7 +447,12 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	ErrExpiredPresignRequest: {
 		Code:           "AccessDenied",
 		Description:    "Request has expired",
-		HTTPStatusCode: http.StatusBadRequest,
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrRequestNotReadyYet: {
+		Code:           "AccessDenied",
+		Description:    "Request is not valid yet",
+		HTTPStatusCode: http.StatusForbidden,
 	},
 	// FIXME: Actual XML error response also contains the header which missed in lsit of signed header parameters.
 	ErrUnsignedHeaders: {
@@ -500,9 +507,14 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Cannot specify more than one suffix rule in a filter.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrFilterPrefixValueInvalid: {
+	ErrFilterValueInvalid: {
 		Code:           "InvalidArgument",
-		Description:    "prefix rule value cannot exceed 1024 characters",
+		Description:    "Size of filter rule value cannot exceed 1024 bytes in UTF-8 representation",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrOverlappingConfigs: {
+		Code:           "InvalidArgument",
+		Description:    "Configurations overlap. Configurations on the same bucket cannot share a common event type.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 

cmd/api-headers.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"
@@ -44,7 +44,7 @@ func generateRequestID() []byte {
 func setCommonHeaders(w http.ResponseWriter) {
 	// Set unique request ID for each reply.
 	w.Header().Set("X-Amz-Request-Id", string(generateRequestID()))
-	w.Header().Set("Server", ("Minio/" + minioReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"))
+	w.Header().Set("Server", ("Minio/" + ReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"))
 	w.Header().Set("Accept-Ranges", "bytes")
 }
 

cmd/api-headers_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"testing"

cmd/api-resources.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"net/url"
@@ -36,7 +36,7 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
 }
 
 // Parse bucket url queries for ListObjects V2.
-func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, maxkeys int, encodingType string) {
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string) {
 	prefix = values.Get("prefix")
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
@@ -47,6 +47,9 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 	}
 	encodingType = values.Get("encoding-type")
 	token = values.Get("continuation-token")
+	if values.Get("fetch-owner") == "true" {
+		fetchOwner = true
+	}
 	return
 }
 

cmd/api-response-multipart.go

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-// File carries any specific responses constructed/necessary in
+// Package cmd file carries any specific responses constructed/necessary in
 // multipart operations.
-package main
+package cmd
 
 import "net/http"
 

cmd/api-response.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"encoding/xml"
@@ -226,16 +226,6 @@ type CompleteMultipartUploadResponse struct {
 	ETag     string
 }
 
-// PostResponse container for completed post upload response
-type PostResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PostResponse" json:"-"`
-
-	Location string
-	Bucket   string
-	Key      string
-	ETag     string
-}
-
 // DeleteError structure.
 type DeleteError struct {
 	Code    string
@@ -337,14 +327,16 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, fetchOwner bool, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
 	var contents []Object
 	var prefixes []CommonPrefix
 	var owner = Owner{}
 	var data = ListObjectsV2Response{}
 
-	owner.ID = "minio"
-	owner.DisplayName = "minio"
+	if fetchOwner {
+		owner.ID = "minio"
+		owner.DisplayName = "minio"
+	}
 
 	for _, object := range resp.Objects {
 		var content = Object{}

cmd/api-router.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import router "github.com/gorilla/mux"
 

cmd/auth-handler.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"
@@ -165,22 +165,18 @@ func setAuthHandler(h http.Handler) http.Handler {
 }
 
 // List of all support S3 auth types.
-var supportedS3AuthTypes = []authType{
-	authTypeAnonymous,
-	authTypePresigned,
-	authTypeSigned,
-	authTypePostPolicy,
-	authTypeStreamingSigned,
+var supportedS3AuthTypes = map[authType]struct{}{
+	authTypeAnonymous:       {},
+	authTypePresigned:       {},
+	authTypeSigned:          {},
+	authTypePostPolicy:      {},
+	authTypeStreamingSigned: {},
 }
 
 // Validate if the authType is valid and supported.
 func isSupportedS3AuthType(aType authType) bool {
-	for _, a := range supportedS3AuthTypes {
-		if a == aType {
-			return true
-		}
-	}
-	return false
+	_, ok := supportedS3AuthTypes[aType]
+	return ok
 }
 
 // handler for validating incoming authorization headers.

cmd/auth-handler_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/benchmark-utils_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/bucket-handlers-listobjects.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"net/http"
@@ -82,7 +82,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		}
 	}
 	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
 
 	// In ListObjectsV2 'continuation-token' is the marker.
 	marker := token
@@ -91,7 +91,8 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		// Then we need to use 'start-after' as marker instead.
 		marker = startAfter
 	}
-	// Validate all the query params before beginning to serve the request.
+	// Validate the query params before beginning to serve the request.
+	// fetch-owner is not validated since it is a boolean
 	if s3Error := listObjectsValidateArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(w, r, s3Error, r.URL.Path)
 		return
@@ -106,7 +107,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, maxKeys, listObjectsInfo)
+	response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, fetchOwner, maxKeys, listObjectsInfo)
 	// Write headers
 	setCommonHeaders(w)
 	// Write success response.

cmd/bucket-handlers.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"encoding/xml"
@@ -23,8 +23,10 @@ import (
 	"net/url"
 	"path"
 	"strings"
+	"sync"
 
 	mux "github.com/gorilla/mux"
+	"github.com/minio/minio-go/pkg/set"
 )
 
 // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
@@ -40,16 +42,16 @@ func enforceBucketPolicy(bucket string, action string, reqURL *url.URL) (s3Error
 	}
 
 	// Construct resource in 'arn:aws:s3:::examplebucket/object' format.
-	resource := AWSResourcePrefix + strings.TrimPrefix(reqURL.Path, "/")
+	resource := AWSResourcePrefix + strings.TrimSuffix(strings.TrimPrefix(reqURL.Path, "/"), "/")
 
 	// Get conditions for policy verification.
-	conditions := make(map[string]string)
+	conditionKeyMap := make(map[string]set.StringSet)
 	for queryParam := range reqURL.Query() {
-		conditions[queryParam] = reqURL.Query().Get(queryParam)
+		conditionKeyMap[queryParam] = set.CreateStringSet(reqURL.Query().Get(queryParam))
 	}
 
 	// Validate action, resource and conditions with current policy statements.
-	if !bucketPolicyEvalStatements(action, resource, conditions, policy.Statements) {
+	if !bucketPolicyEvalStatements(action, resource, conditionKeyMap, policy.Statements) {
 		return ErrAccessDenied
 	}
 	return ErrNone
@@ -239,24 +241,47 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var deleteErrors []DeleteError
-	var deletedObjects []ObjectIdentifier
-	// Loop through all the objects and delete them sequentially.
-	for _, object := range deleteObjects.Objects {
-		err := api.ObjectAPI.DeleteObject(bucket, object.ObjectName)
-		if err == nil {
-			deletedObjects = append(deletedObjects, ObjectIdentifier{
-				ObjectName: object.ObjectName,
-			})
-		} else {
-			errorIf(err, "Unable to delete object.")
-			deleteErrors = append(deleteErrors, DeleteError{
-				Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
-				Message: errorCodeResponse[toAPIErrorCode(err)].Description,
-				Key:     object.ObjectName,
-			})
-		}
+	var wg = &sync.WaitGroup{} // Allocate a new wait group.
+	var dErrs = make([]error, len(deleteObjects.Objects))
+
+	// Delete all requested objects in parallel.
+	for index, object := range deleteObjects.Objects {
+		wg.Add(1)
+		go func(i int, obj ObjectIdentifier) {
+			defer wg.Done()
+			dErr := api.ObjectAPI.DeleteObject(bucket, obj.ObjectName)
+			if dErr != nil {
+				dErrs[i] = dErr
+			}
+		}(index, object)
 	}
+	wg.Wait()
+
+	// Collect deleted objects and errors if any.
+	var deletedObjects []ObjectIdentifier
+	var deleteErrors []DeleteError
+	for index, err := range dErrs {
+		object := deleteObjects.Objects[index]
+		// Success deleted objects are collected separately.
+		if err == nil {
+			deletedObjects = append(deletedObjects, object)
+			continue
+		}
+		if _, ok := err.(ObjectNotFound); ok {
+			// If the object is not found it should be
+			// accounted as deleted as per S3 spec.
+			deletedObjects = append(deletedObjects, object)
+			continue
+		}
+		errorIf(err, "Unable to delete object. %s", object.ObjectName)
+		// Error during delete should be collected separately.
+		deleteErrors = append(deleteErrors, DeleteError{
+			Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
+			Message: errorCodeResponse[toAPIErrorCode(err)].Description,
+			Key:     object.ObjectName,
+		})
+	}
+
 	// Generate response
 	response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
 	encodedSuccessResponse := encodeResponse(response)
@@ -264,6 +289,22 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	setCommonHeaders(w)
 	// Write success response.
 	writeSuccessResponse(w, encodedSuccessResponse)
+
+	if globalEventNotifier.IsBucketNotificationSet(bucket) {
+		// Notify deleted event for objects.
+		for _, dobj := range deletedObjects {
+			eventNotify(eventData{
+				Type:   ObjectRemovedDelete,
+				Bucket: bucket,
+				ObjInfo: ObjectInfo{
+					Name: dobj.ObjectName,
+				},
+				ReqParams: map[string]string{
+					"sourceIPAddress": r.RemoteAddr,
+				},
+			})
+		}
+	}
 }
 
 // PutBucketHandler - PUT Bucket
@@ -283,6 +324,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	// requests which do not follow valid region requirements.
 	if s3Error := isValidLocationConstraint(r); s3Error != ErrNone {
 		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
 	}
 
 	// Proceed to creating a bucket.
@@ -351,25 +393,24 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if md5Sum != "" {
 		w.Header().Set("ETag", "\""+md5Sum+"\"")
 	}
-	encodedSuccessResponse := encodeResponse(PostResponse{
-		Location: getObjectLocation(bucket, object), // TODO Full URL is preferred
-		Bucket:   bucket,
-		Key:      object,
-		ETag:     md5Sum,
-	})
 
+	// TODO full URL is preferred.
+	w.Header().Set("Location", getObjectLocation(bucket, object))
+
+	// Set common headers.
 	setCommonHeaders(w)
 
-	writeSuccessResponse(w, encodedSuccessResponse)
+	// Write successful response.
+	writeSuccessNoContent(w)
 
-	// Fetch object info for notifications.
-	objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
-	if err != nil {
-		errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
-		return
-	}
+	if globalEventNotifier.IsBucketNotificationSet(bucket) {
+		// Fetch object info for notifications.
+		objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
+		if err != nil {
+			errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
+			return
+		}
 
-	if eventN.IsBucketNotificationSet(bucket) {
 		// Notify object created event.
 		eventNotify(eventData{
 			Type:    ObjectCreatedPost,

cmd/bucket-notification-datatypes.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"encoding/xml"
@@ -32,33 +32,30 @@ type keyFilter struct {
 	FilterRules []filterRule `xml:"FilterRule,omitempty"`
 }
 
-// Queue SQS configuration.
-type queueConfig struct {
+// Common elements of service notification.
+type serviceConfig struct {
 	Events []string `xml:"Event"`
 	Filter struct {
 		Key keyFilter `xml:"S3Key,omitempty"`
 	}
-	ID       string `xml:"Id"`
+	ID string `xml:"Id"`
+}
+
+// Queue SQS configuration.
+type queueConfig struct {
+	serviceConfig
 	QueueARN string `xml:"Queue"`
 }
 
 // Topic SNS configuration, this is a compliance field not used by minio yet.
 type topicConfig struct {
-	Events []string `xml:"Event"`
-	Filter struct {
-		Key keyFilter `xml:"S3Key"`
-	}
-	ID       string `xml:"Id"`
+	serviceConfig
 	TopicARN string `xml:"Topic"`
 }
 
 // Lambda function configuration, this is a compliance field not used by minio yet.
 type lambdaConfig struct {
-	Events []string `xml:"Event"`
-	Filter struct {
-		Key keyFilter `xml:"S3Key,omitempty"`
-	}
-	ID        string `xml:"Id"`
+	serviceConfig
 	LambdaARN string `xml:"CloudFunction"`
 }
 

cmd/bucket-notification-handlers.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"
@@ -88,7 +88,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	_, err := api.ObjectAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to bucket info.")
+		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
 		return
 	}
@@ -141,7 +141,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	}
 
 	// Set bucket notification config.
-	eventN.SetBucketNotificationConfig(bucket, &notificationCfg)
+	globalEventNotifier.SetBucketNotificationConfig(bucket, &notificationCfg)
 
 	// Success.
 	writeSuccessResponse(w, nil)
@@ -227,7 +227,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	notificationCfg := eventN.GetBucketNotificationConfig(bucket)
+	notificationCfg := globalEventNotifier.GetBucketNotificationConfig(bucket)
 	if notificationCfg == nil {
 		writeErrorResponse(w, r, ErrARNNotification, r.URL.Path)
 		return
@@ -249,9 +249,9 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	defer close(nEventCh)
 
 	// Set sns target.
-	eventN.SetSNSTarget(topicARN, nEventCh)
+	globalEventNotifier.SetSNSTarget(topicARN, nEventCh)
 	// Remove sns listener after the writer has closed or the client disconnected.
-	defer eventN.RemoveSNSTarget(topicARN, nEventCh)
+	defer globalEventNotifier.RemoveSNSTarget(topicARN, nEventCh)
 
 	// Start sending bucket notifications.
 	sendBucketNotification(w, nEventCh)

cmd/bucket-notification-handlers_test.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"bytes"

cmd/bucket-notification-utils.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "strings"
 
@@ -88,9 +88,8 @@ func checkFilterRules(filterRules []filterRule) APIErrorCode {
 			}
 		}
 
-		// Maximum prefix length can be up to 1,024 characters, validate.
 		if !IsValidObjectPrefix(filterRule.Value) {
-			return ErrFilterPrefixValueInvalid
+			return ErrFilterValueInvalid
 		}
 
 		// Set the new rule name to keep track of duplicates.
@@ -100,26 +99,36 @@ func checkFilterRules(filterRules []filterRule) APIErrorCode {
 	return ErrNone
 }
 
-// checkQueueARN - check if the queue arn is valid.
-func checkQueueARN(queueARN string) APIErrorCode {
-	if !strings.HasPrefix(queueARN, minioSqs) {
+// Checks validity of input ARN for a given arnType.
+func checkARN(arn, arnType string) APIErrorCode {
+	if !strings.HasPrefix(arn, arnType) {
 		return ErrARNNotification
 	}
-	if !strings.HasPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":") {
+	if !strings.HasPrefix(arn, arnType+serverConfig.GetRegion()+":") {
 		return ErrRegionNotification
 	}
+	account := strings.SplitN(strings.TrimPrefix(arn, arnType+serverConfig.GetRegion()+":"), ":", 2)
+	switch len(account) {
+	case 1:
+		// This means ARN is malformed, account should have min of 2elements.
+		return ErrARNNotification
+	case 2:
+		// Account topic id or topic name cannot be empty.
+		if account[0] == "" || account[1] == "" {
+			return ErrARNNotification
+		}
+	}
 	return ErrNone
 }
 
+// checkQueueARN - check if the queue arn is valid.
+func checkQueueARN(queueARN string) APIErrorCode {
+	return checkARN(queueARN, minioSqs)
+}
+
 // checkTopicARN - check if the topic arn is valid.
 func checkTopicARN(topicARN string) APIErrorCode {
-	if !strings.HasPrefix(topicARN, minioTopic) {
-		return ErrARNNotification
-	}
-	if !strings.HasPrefix(topicARN, minioTopic+serverConfig.GetRegion()+":") {
-		return ErrRegionNotification
-	}
-	return ErrNone
+	return checkARN(topicARN, minioTopic)
 }
 
 // Returns true if the topicARN is for an Minio sns listen type.
@@ -257,17 +266,68 @@ func validateTopicConfigs(topicConfigs []topicConfig) APIErrorCode {
 	return ErrNone
 }
 
+// Check all the queue configs for any duplicates.
+func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
+	configMaps := make(map[string]int)
+
+	// Navigate through each configs and count the entries.
+	for _, config := range configs {
+		configMaps[config.QueueARN]++
+	}
+
+	// Validate if there are any duplicate counts.
+	for _, count := range configMaps {
+		if count != 1 {
+			return ErrOverlappingConfigs
+		}
+	}
+
+	// Success.
+	return ErrNone
+}
+
+// Check all the topic configs for any duplicates.
+func checkDuplicateTopicConfigs(configs []topicConfig) APIErrorCode {
+	configMaps := make(map[string]int)
+
+	// Navigate through each configs and count the entries.
+	for _, config := range configs {
+		configMaps[config.TopicARN]++
+	}
+
+	// Validate if there are any duplicate counts.
+	for _, count := range configMaps {
+		if count != 1 {
+			return ErrOverlappingConfigs
+		}
+	}
+
+	// Success.
+	return ErrNone
+}
+
 // Validates all the bucket notification configuration for their validity,
 // if one of the config is malformed or has invalid data it is rejected.
 // Configuration is never applied partially.
 func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
+	// Validate all queue configs.
 	if s3Error := validateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone {
 		return s3Error
 	}
+	// Validate all topic configs.
 	if s3Error := validateTopicConfigs(nConfig.TopicConfigs); s3Error != ErrNone {
 		return s3Error
 	}
 
+	// Check for duplicate queue configs.
+	if s3Error := checkDuplicateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone {
+		return s3Error
+	}
+	// Check for duplicate topic configs.
+	if s3Error := checkDuplicateTopicConfigs(nConfig.TopicConfigs); s3Error != ErrNone {
+		return s3Error
+	}
+
 	// Add validation for other configurations.
 	return ErrNone
 }

cmd/bucket-notification-utils_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "testing"
 
@@ -119,18 +119,49 @@ func TestTopicARN(t *testing.T) {
 			topicARN: "arn:minio:sns:us-east-1:10:minio",
 			errCode:  ErrNone,
 		},
-		// Invalid empty queue arn.
+		// Invalid empty topic  arn.
 		{
 			topicARN: "",
 			errCode:  ErrARNNotification,
 		},
+		// Invalid notification service type.
+		{
+			topicARN: "arn:minio:sqs:us-east-1:1:listen",
+			errCode:  ErrARNNotification,
+		},
 		// Invalid region 'us-west-1' in queue arn.
 		{
-			topicARN: "arn:minio:sns:us-west-1:1:redis",
+			topicARN: "arn:minio:sns:us-west-1:1:listen",
 			errCode:  ErrRegionNotification,
 		},
+		// Empty topic account id is invalid.
+		{
+			topicARN: "arn:minio:sns:us-east-1::listen",
+			errCode:  ErrARNNotification,
+		},
+		// Empty topic account name is invalid.
+		{
+			topicARN: "arn:minio:sns:us-east-1:10:",
+			errCode:  ErrARNNotification,
+		},
+		// Empty topic account id and account name is invalid.
+		{
+			topicARN: "arn:minio:sns:us-east-1::",
+			errCode:  ErrARNNotification,
+		},
+		// Missing topic id and separator missing at the end in topic arn.
+		{
+			topicARN: "arn:minio:sns:us-east-1:listen",
+			errCode:  ErrARNNotification,
+		},
+		// Missing topic id and empty string at the end in topic arn.
+		{
+			topicARN: "arn:minio:sns:us-east-1:",
+			errCode:  ErrARNNotification,
+		},
 	}
 
+	// Validate all topics.
 	for i, testCase := range testCases {
 		errCode := checkTopicARN(testCase.topicARN)
 		if testCase.errCode != errCode {
@@ -171,13 +202,44 @@ func TestQueueARN(t *testing.T) {
 			queueARN: "",
 			errCode:  ErrARNNotification,
 		},
+		// Invalid notification service type.
+		{
+			queueARN: "arn:minio:sns:us-east-1:1:listen",
+			errCode:  ErrARNNotification,
+		},
 		// Invalid region 'us-west-1' in queue arn.
 		{
 			queueARN: "arn:minio:sqs:us-west-1:1:redis",
 			errCode:  ErrRegionNotification,
 		},
+		// Invalid queue name empty in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid queue id empty in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1::redis",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid queue id and queue name empty in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1::",
+			errCode:  ErrARNNotification,
+		},
+		// Missing queue id and separator missing at the end in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:amqp",
+			errCode:  ErrARNNotification,
+		},
+		// Missing queue id and empty string at the end in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:",
+			errCode:  ErrARNNotification,
+		},
 	}
 
+	// Validate all tests for queue arn.
 	for i, testCase := range testCases {
 		errCode := checkQueueARN(testCase.queueARN)
 		if testCase.errCode != errCode {

cmd/bucket-policy-handlers.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"
@@ -24,6 +24,7 @@ import (
 	"net/http"
 
 	mux "github.com/gorilla/mux"
+	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
@@ -32,7 +33,7 @@ const maxAccessPolicySize = 20 * 1024 // 20KiB.
 
 // Verify if a given action is valid for the url path based on the
 // existing bucket access policy.
-func bucketPolicyEvalStatements(action string, resource string, conditions map[string]string, statements []policyStatement) bool {
+func bucketPolicyEvalStatements(action string, resource string, conditions map[string]set.StringSet, statements []policyStatement) bool {
 	for _, statement := range statements {
 		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
 			if statement.Effect == "Allow" {
@@ -48,7 +49,7 @@ func bucketPolicyEvalStatements(action string, resource string, conditions map[s
 }
 
 // Verify if action, resource and conditions match input policy statement.
-func bucketPolicyMatchStatement(action string, resource string, conditions map[string]string, statement policyStatement) bool {
+func bucketPolicyMatchStatement(action string, resource string, conditions map[string]set.StringSet, statement policyStatement) bool {
 	// Verify if action matches.
 	if bucketPolicyActionMatch(action, statement) {
 		// Verify if resource matches.
@@ -64,40 +65,30 @@ func bucketPolicyMatchStatement(action string, resource string, conditions map[s
 
 // Verify if given action matches with policy statement.
 func bucketPolicyActionMatch(action string, statement policyStatement) bool {
-	for _, policyAction := range statement.Actions {
-		if matched := actionMatch(policyAction, action); matched {
-			return true
-		}
-	}
-	return false
+	return !statement.Actions.FuncMatch(actionMatch, action).IsEmpty()
 }
 
 // Match function matches wild cards in 'pattern' for resource.
 func resourceMatch(pattern, resource string) bool {
-	return wildcard.MatchExtended(pattern, resource)
+	return wildcard.Match(pattern, resource)
 }
 
 // Match function matches wild cards in 'pattern' for action.
 func actionMatch(pattern, action string) bool {
-	return wildcard.Match(pattern, action)
+	return wildcard.MatchSimple(pattern, action)
 }
 
 // Verify if given resource matches with policy statement.
 func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
-	for _, resourcep := range statement.Resources {
-		// the resource rule for object could contain "*" wild card.
-		// the requested object can be given access based on the already set bucket policy if
-		// the match is successful.
-		// More info: http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html .
-		if matched := resourceMatch(resourcep, resource); matched {
-			return true
-		}
-	}
-	return false
+	// the resource rule for object could contain "*" wild card.
+	// the requested object can be given access based on the already set bucket policy if
+	// the match is successful.
+	// More info: http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html.
+	return !statement.Resources.FuncMatch(resourceMatch, resource).IsEmpty()
 }
 
 // Verify if given condition matches with policy statement.
-func bucketPolicyConditionMatch(conditions map[string]string, statement policyStatement) bool {
+func bucketPolicyConditionMatch(conditions map[string]set.StringSet, statement policyStatement) bool {
 	// Supports following conditions.
 	// - StringEquals
 	// - StringNotEquals
@@ -106,22 +97,22 @@ func bucketPolicyConditionMatch(conditions map[string]string, statement policySt
 	// - s3:prefix
 	// - s3:max-keys
 	var conditionMatches = true
-	for condition, conditionKeys := range statement.Conditions {
+	for condition, conditionKeyVal := range statement.Conditions {
 		if condition == "StringEquals" {
-			if conditionKeys["s3:prefix"] != conditions["prefix"] {
+			if !conditionKeyVal["s3:prefix"].Equals(conditions["prefix"]) {
 				conditionMatches = false
 				break
 			}
-			if conditionKeys["s3:max-keys"] != conditions["max-keys"] {
+			if !conditionKeyVal["s3:max-keys"].Equals(conditions["max-keys"]) {
 				conditionMatches = false
 				break
 			}
 		} else if condition == "StringNotEquals" {
-			if conditionKeys["s3:prefix"] == conditions["prefix"] {
+			if !conditionKeyVal["s3:prefix"].Equals(conditions["prefix"]) {
 				conditionMatches = false
 				break
 			}
-			if conditionKeys["s3:max-keys"] == conditions["max-keys"] {
+			if !conditionKeyVal["s3:max-keys"].Equals(conditions["max-keys"]) {
 				conditionMatches = false
 				break
 			}

cmd/bucket-policy-handlers_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"
@@ -23,6 +23,8 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"testing"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
 // Tests validate Bucket policy resource matcher.
@@ -31,7 +33,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
 	// generates statement with given resource..
 	generateStatement := func(resource string) policyStatement {
 		statement := policyStatement{}
-		statement.Resources = []string{resource}
+		statement.Resources = set.CreateStringSet([]string{resource}...)
 		return statement
 	}
 
@@ -336,7 +338,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrH
 	credentials := serverConfig.GetCredential()
 
 	// template for constructing HTTP request body for PUT bucket policy.
-	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}`
+	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"],"Sid":""}]}`
 
 	// Writing bucket policy before running test on GetBucketPolicy.
 	putTestPolicies := []struct {

cmd/bucket-policy-migrate.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/bucket-policy-parser.go

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-// This file implements AWS Access Policy Language parser in
+// Package cmd This file implements AWS Access Policy Language parser in
 // accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
-package main
+package cmd
 
 import (
 	"encoding/json"
@@ -26,6 +26,8 @@ import (
 	"path"
 	"sort"
 	"strings"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
 const (
@@ -34,50 +36,39 @@ const (
 )
 
 // supportedActionMap - lists all the actions supported by minio.
-var supportedActionMap = map[string]struct{}{
-	"*":                             {},
-	"s3:*":                          {},
-	"s3:GetObject":                  {},
-	"s3:ListBucket":                 {},
-	"s3:PutObject":                  {},
-	"s3:GetBucketLocation":          {},
-	"s3:DeleteObject":               {},
-	"s3:AbortMultipartUpload":       {},
-	"s3:ListBucketMultipartUploads": {},
-	"s3:ListMultipartUploadParts":   {},
-}
+var supportedActionMap = set.CreateStringSet("*", "*", "s3:*", "s3:GetObject",
+	"s3:ListBucket", "s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
+	"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts")
 
 // supported Conditions type.
-var supportedConditionsType = map[string]struct{}{
-	"StringEquals":    {},
-	"StringNotEquals": {},
-}
+var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals")
 
 // Validate s3:prefix, s3:max-keys are present if not
 // supported keys for the conditions.
-var supportedConditionsKey = map[string]struct{}{
-	"s3:prefix":   {},
-	"s3:max-keys": {},
-}
+var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys")
 
-// User - canonical users list.
+// supportedEffectMap - supported effects.
+var supportedEffectMap = set.CreateStringSet("Allow", "Deny")
+
+// policyUser - canonical users list.
 type policyUser struct {
-	AWS []string
+	AWS           set.StringSet `json:"AWS,omitempty"`
+	CanonicalUser set.StringSet `json:"CanonicalUser,omitempty"`
 }
 
 // Statement - minio policy statement
 type policyStatement struct {
-	Sid        string
+	Actions    set.StringSet                       `json:"Action"`
+	Conditions map[string]map[string]set.StringSet `json:"Condition,omitempty"`
 	Effect     string
-	Principal  policyUser                   `json:"Principal"`
-	Actions    []string                     `json:"Action"`
-	Resources  []string                     `json:"Resource"`
-	Conditions map[string]map[string]string `json:"Condition,omitempty"`
+	Principal  policyUser    `json:"Principal"`
+	Resources  set.StringSet `json:"Resource"`
+	Sid        string
 }
 
 // bucketPolicy - collection of various bucket policy statements.
 type bucketPolicy struct {
-	Version    string            // date in 0000-00-00 format
+	Version    string            // date in YYYY-MM-DD format
 	Statements []policyStatement `json:"Statement"`
 }
 
@@ -91,51 +82,42 @@ func (b bucketPolicy) String() string {
 	return string(bbytes)
 }
 
-// supportedEffectMap - supported effects.
-var supportedEffectMap = map[string]struct{}{
-	"Allow": {},
-	"Deny":  {},
-}
-
 // isValidActions - are actions valid.
-func isValidActions(actions []string) (err error) {
+func isValidActions(actions set.StringSet) (err error) {
 	// Statement actions cannot be empty.
 	if len(actions) == 0 {
 		err = errors.New("Action list cannot be empty.")
 		return err
 	}
-	for _, action := range actions {
-		if _, ok := supportedActionMap[action]; !ok {
-			err = errors.New("Unsupported action found: ‘" + action + "’, please validate your policy document.")
-			return err
-		}
+	if unsupportedActions := actions.Difference(supportedActionMap); !unsupportedActions.IsEmpty() {
+		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document.", unsupportedActions)
+		return err
 	}
 	return nil
 }
 
 // isValidEffect - is effect valid.
-func isValidEffect(effect string) error {
+func isValidEffect(effect string) (err error) {
 	// Statement effect cannot be empty.
-	if len(effect) == 0 {
-		err := errors.New("Policy effect cannot be empty.")
+	if effect == "" {
+		err = errors.New("Policy effect cannot be empty.")
 		return err
 	}
-	_, ok := supportedEffectMap[effect]
-	if !ok {
-		err := errors.New("Unsupported Effect found: ‘" + effect + "’, please validate your policy document.")
+	if !supportedEffectMap.Contains(effect) {
+		err = errors.New("Unsupported Effect found: ‘" + effect + "’, please validate your policy document.")
 		return err
 	}
 	return nil
 }
 
 // isValidResources - are valid resources.
-func isValidResources(resources []string) (err error) {
+func isValidResources(resources set.StringSet) (err error) {
 	// Statement resources cannot be empty.
 	if len(resources) == 0 {
 		err = errors.New("Resource list cannot be empty.")
 		return err
 	}
-	for _, resource := range resources {
+	for resource := range resources {
 		if !strings.HasPrefix(resource, AWSResourcePrefix) {
 			err = errors.New("Unsupported resource style found: ‘" + resource + "’, please validate your policy document.")
 			return err
@@ -150,63 +132,50 @@ func isValidResources(resources []string) (err error) {
 }
 
 // isValidPrincipals - are valid principals.
-func isValidPrincipals(principals []string) (err error) {
+func isValidPrincipals(principals set.StringSet) (err error) {
 	// Statement principal should have a value.
 	if len(principals) == 0 {
 		err = errors.New("Principal cannot be empty.")
 		return err
 	}
-	for _, principal := range principals {
+	if unsuppPrincipals := principals.Difference(set.CreateStringSet([]string{"*"}...)); !unsuppPrincipals.IsEmpty() {
 		// Minio does not support or implement IAM, "*" is the only valid value.
 		// Amazon s3 doc on principals: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
-		if principal != "*" {
-			err = fmt.Errorf("Unsupported principal style found: ‘%s’, please validate your policy document.", principal)
-			return err
-		}
+		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document.", unsuppPrincipals)
+		return err
 	}
 	return nil
 }
 
 // isValidConditions - are valid conditions.
-func isValidConditions(conditions map[string]map[string]string) (err error) {
-	// Returns true if string 'a' is found in the list.
-	findString := func(a string, list []string) bool {
-		for _, b := range list {
-			if b == a {
-				return true
-			}
-		}
-		return false
-	}
-	conditionKeyVal := make(map[string][]string)
+func isValidConditions(conditions map[string]map[string]set.StringSet) (err error) {
 	// Verify conditions should be valid.
 	// Validate if stringEquals, stringNotEquals are present
 	// if not throw an error.
+	conditionKeyVal := make(map[string]set.StringSet)
 	for conditionType := range conditions {
-		_, validType := supportedConditionsType[conditionType]
-		if !validType {
+		if !supportedConditionsType.Contains(conditionType) {
 			err = fmt.Errorf("Unsupported condition type '%s', please validate your policy document.", conditionType)
 			return err
 		}
-		for key := range conditions[conditionType] {
-			_, validKey := supportedConditionsKey[key]
-			if !validKey {
+		for key, value := range conditions[conditionType] {
+			if !supportedConditionsKey.Contains(key) {
 				err = fmt.Errorf("Unsupported condition key '%s', please validate your policy document.", conditionType)
 				return err
 			}
-			conditionArray, ok := conditionKeyVal[key]
-			if ok && findString(conditions[conditionType][key], conditionArray) {
+			conditionVal, ok := conditionKeyVal[key]
+			if ok && !value.Intersection(conditionVal).IsEmpty() {
 				err = fmt.Errorf("Ambigious condition values for key '%s', please validate your policy document.", key)
 				return err
 			}
-			conditionKeyVal[key] = append(conditionKeyVal[key], conditions[conditionType][key])
+			conditionKeyVal[key] = value
 		}
 	}
 	return nil
 }
 
 // List of actions for which prefixes are not allowed.
-var invalidPrefixActions = map[string]struct{}{
+var invalidPrefixActions = set.StringSet{
 	"s3:GetBucketLocation":          {},
 	"s3:ListBucket":                 {},
 	"s3:ListBucketMultipartUploads": {},
@@ -227,10 +196,10 @@ func resourcePrefix(resource string) string {
 func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIErrorCode {
 	// Validate statements for special actions and collect resources
 	// for others to validate nesting.
-	var resourceMap = make(map[string]struct{})
+	var resourceMap = set.NewStringSet()
 	for _, statement := range bucketPolicy.Statements {
-		for _, action := range statement.Actions {
-			for _, resource := range statement.Resources {
+		for action := range statement.Actions {
+			for resource := range statement.Resources {
 				resourcePrefix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
 				if _, ok := invalidPrefixActions[action]; ok {
 					// Resource prefix is not equal to bucket for
@@ -245,7 +214,7 @@ func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIEr
 						return ErrMalformedPolicy
 					}
 					// All valid resources collect them separately to verify nesting.
-					resourceMap[resourcePrefix] = struct{}{}
+					resourceMap.Add(resourcePrefix)
 				}
 			}
 		}

cmd/bucket-policy-parser_test.go

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package main
+package cmd
 
 import (
 	"bytes"
@@ -21,8 +21,11 @@ import (
 	"errors"
 	"fmt"
 	"testing"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
+// Common bucket actions for both read and write policies.
 var (
 	readWriteBucketActions = []string{
 		"s3:GetBucketLocation",
@@ -73,9 +76,9 @@ var (
 func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatement {
 	objectResourceStatement := policyStatement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal.AWS = []string{"*"}
-	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
-	objectResourceStatement.Actions = readWriteObjectActions
+	objectResourceStatement.Principal.AWS = set.CreateStringSet([]string{"*"}...)
+	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...)
+	objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...)
 	return objectResourceStatement
 }
 
@@ -83,9 +86,9 @@ func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatemen
 func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatement {
 	bucketResourceStatement := policyStatement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal.AWS = []string{"*"}
-	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
-	bucketResourceStatement.Actions = readWriteBucketActions
+	bucketResourceStatement.Principal.AWS = set.CreateStringSet([]string{"*"}...)
+	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...)
+	bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...)
 	return bucketResourceStatement
 }
 
@@ -101,9 +104,9 @@ func getReadWriteStatement(bucketName, objectPrefix string) []policyStatement {
 func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
 	bucketResourceStatement := policyStatement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal.AWS = []string{"*"}
-	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
-	bucketResourceStatement.Actions = readOnlyBucketActions
+	bucketResourceStatement.Principal.AWS = set.CreateStringSet([]string{"*"}...)
+	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...)
+	bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...)
 	return bucketResourceStatement
 }
 
@@ -111,9 +114,9 @@ func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement
 func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
 	objectResourceStatement := policyStatement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal.AWS = []string{"*"}
-	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
-	objectResourceStatement.Actions = readOnlyObjectActions
+	objectResourceStatement.Principal.AWS = set.CreateStringSet([]string{"*"}...)
+	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...)
+	objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...)
 	return objectResourceStatement
 }
 
@@ -130,9 +133,9 @@ func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatemen
 
 	bucketResourceStatement := policyStatement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal.AWS = []string{"*"}
-	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
-	bucketResourceStatement.Actions = writeOnlyBucketActions
+	bucketResourceStatement.Principal.AWS = set.CreateStringSet([]string{"*"}...)
+	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...)
+	bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...)
 	return bucketResourceStatement
 }
 
@@ -140,9 +143,9 @@ func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatemen
 func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
 	objectResourceStatement := policyStatement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal.AWS = []string{"*"}
-	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
-	objectResourceStatement.Actions = writeOnlyObjectActions
+	objectResourceStatement.Principal.AWS = set.CreateStringSet([]string{"*"}...)
+	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...)
+	objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...)
 	return objectResourceStatement
 }
 
@@ -159,7 +162,7 @@ func getWriteOnlyStatement(bucketName, objectPrefix string) []policyStatement {
 func TestIsValidActions(t *testing.T) {
 	testCases := []struct {
 		// input.
-		actions []string
+		actions set.StringSet
 		// expected output.
 		err error
 		// flag indicating whether the test should pass.
@@ -168,19 +171,22 @@ func TestIsValidActions(t *testing.T) {
 		// Inputs with unsupported Action.
 		// Test case - 1.
 		// "s3:ListObject" is an invalid Action.
-		{[]string{"s3:GetObject", "s3:ListObject", "s3:RemoveObject"}, errors.New("Unsupported action found: ‘s3:ListObject’, please validate your policy document."), false},
+		{set.CreateStringSet([]string{"s3:GetObject", "s3:ListObject", "s3:RemoveObject"}...),
+			errors.New("Unsupported actions found: ‘set.StringSet{\"s3:RemoveObject\":struct {}{}, \"s3:ListObject\":struct {}{}}’, please validate your policy document."), false},
 		// Test case - 2.
 		// Empty Actions.
-		{[]string{}, errors.New("Action list cannot be empty."), false},
+		{set.CreateStringSet([]string{}...), errors.New("Action list cannot be empty."), false},
 		// Test case - 3.
 		// "s3:DeleteEverything"" is an invalid Action.
-		{[]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"},
-			errors.New("Unsupported action found: ‘s3:DeleteEverything’, please validate your policy document."), false},
-
+		{set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...),
+			errors.New("Unsupported actions found: ‘set.StringSet{\"s3:DeleteEverything\":struct {}{}}’, please validate your policy document."), false},
 		// Inputs with valid Action.
 		// Test Case - 4.
-		{[]string{"s3:*", "*", "s3:GetObject", "s3:ListBucket",
-			"s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts"}, nil, true},
+		{set.CreateStringSet([]string{
+			"s3:*", "*", "s3:GetObject", "s3:ListBucket",
+			"s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
+			"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads",
+			"s3:ListMultipartUploadParts"}...), nil, true},
 	}
 	for i, testCase := range testCases {
 		err := isValidActions(testCase.actions)
@@ -190,13 +196,6 @@ func TestIsValidActions(t *testing.T) {
 		if err == nil && !testCase.shouldPass {
 			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
 		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-
 	}
 }
 
@@ -212,16 +211,18 @@ func TestIsValidEffect(t *testing.T) {
 	}{
 		// Inputs with unsupported Effect.
 		// Test case - 1.
-		{"DontAllow", errors.New("Unsupported Effect found: ‘DontAllow’, please validate your policy document."), false},
+		{"", errors.New("Policy effect cannot be empty."), false},
 		// Test case - 2.
-		{"NeverAllow", errors.New("Unsupported Effect found: ‘NeverAllow’, please validate your policy document."), false},
+		{"DontAllow", errors.New("Unsupported Effect found: ‘DontAllow’, please validate your policy document."), false},
 		// Test case - 3.
+		{"NeverAllow", errors.New("Unsupported Effect found: ‘NeverAllow’, please validate your policy document."), false},
+		// Test case - 4.
 		{"AllowAlways", errors.New("Unsupported Effect found: ‘AllowAlways’, please validate your policy document."), false},
 
 		// Inputs with valid Effect.
-		// Test Case - 4.
-		{"Allow", nil, true},
 		// Test Case - 5.
+		{"Allow", nil, true},
+		// Test Case - 6.
 		{"Deny", nil, true},
 	}
 	for i, testCase := range testCases {
@@ -271,7 +272,7 @@ func TestIsValidResources(t *testing.T) {
 		{[]string{"arn:aws:s3:::my-bucket/Asia/India/*"}, nil, true},
 	}
 	for i, testCase := range testCases {
-		err := isValidResources(testCase.resources)
+		err := isValidResources(set.CreateStringSet(testCase.resources...))
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
 		}
@@ -303,16 +304,15 @@ func TestIsValidPrincipals(t *testing.T) {
 		{[]string{}, errors.New("Principal cannot be empty."), false},
 		// Test case - 2.
 		// "*" is the only valid principal.
-		{[]string{"my-principal"}, errors.New("Unsupported principal style found: ‘my-principal’, please validate your policy document."), false},
+		{[]string{"my-principal"}, errors.New("Unsupported principals found: ‘set.StringSet{\"my-principal\":struct {}{}}’, please validate your policy document."), false},
 		// Test case - 3.
-		{[]string{"*", "111122233"}, errors.New("Unsupported principal style found: ‘111122233’, please validate your policy document."), false},
-
+		{[]string{"*", "111122233"}, errors.New("Unsupported principals found: ‘set.StringSet{\"111122233\":struct {}{}}’, please validate your policy document."), false},
 		// Test case - 4.
 		// Test case with valid principal value.
 		{[]string{"*"}, nil, true},
 	}
 	for i, testCase := range testCases {
-		err := isValidPrincipals(testCase.principals)
+		err := isValidPrincipals(set.CreateStringSet(testCase.principals...))
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
 		}
@@ -331,70 +331,70 @@ func TestIsValidPrincipals(t *testing.T) {
 // Tests validate policyStatement condition validator.
 func TestIsValidConditions(t *testing.T) {
 	// returns empty conditions map.
-	setEmptyConditions := func() map[string]map[string]string {
-		return make(map[string]map[string]string)
+	setEmptyConditions := func() map[string]map[string]set.StringSet {
+		return make(map[string]map[string]set.StringSet)
 	}
 
 	// returns map with the "StringEquals" set to empty map.
-	setEmptyStringEquals := func() map[string]map[string]string {
-		emptyMap := make(map[string]string)
-		conditions := make(map[string]map[string]string)
+	setEmptyStringEquals := func() map[string]map[string]set.StringSet {
+		emptyMap := make(map[string]set.StringSet)
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions["StringEquals"] = emptyMap
 		return conditions
 
 	}
 
 	// returns map with the "StringNotEquals" set to empty map.
-	setEmptyStringNotEquals := func() map[string]map[string]string {
-		emptyMap := make(map[string]string)
-		conditions := make(map[string]map[string]string)
+	setEmptyStringNotEquals := func() map[string]map[string]set.StringSet {
+		emptyMap := make(map[string]set.StringSet)
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions["StringNotEquals"] = emptyMap
 		return conditions
 
 	}
 	// Generate conditions.
-	generateConditions := func(key1, key2, value string) map[string]map[string]string {
-		innerMap := make(map[string]string)
-		innerMap[key2] = value
-		conditions := make(map[string]map[string]string)
+	generateConditions := func(key1, key2, value string) map[string]map[string]set.StringSet {
+		innerMap := make(map[string]set.StringSet)
+		innerMap[key2] = set.CreateStringSet(value)
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions[key1] = innerMap
 		return conditions
 	}
 
 	// generate ambigious conditions.
-	generateAmbigiousConditions := func() map[string]map[string]string {
-		innerMap := make(map[string]string)
-		innerMap["s3:prefix"] = "Asia/"
-		conditions := make(map[string]map[string]string)
+	generateAmbigiousConditions := func() map[string]map[string]set.StringSet {
+		innerMap := make(map[string]set.StringSet)
+		innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions["StringEquals"] = innerMap
 		conditions["StringNotEquals"] = innerMap
 		return conditions
 	}
 
 	// generate valid and non valid type in the condition map.
-	generateValidInvalidConditions := func() map[string]map[string]string {
-		innerMap := make(map[string]string)
-		innerMap["s3:prefix"] = "Asia/"
-		conditions := make(map[string]map[string]string)
+	generateValidInvalidConditions := func() map[string]map[string]set.StringSet {
+		innerMap := make(map[string]set.StringSet)
+		innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions["StringEquals"] = innerMap
 		conditions["InvalidType"] = innerMap
 		return conditions
 	}
 
 	// generate valid and invalid keys for valid types in the same condition map.
-	generateValidInvalidConditionKeys := func() map[string]map[string]string {
-		innerMapValid := make(map[string]string)
-		innerMapValid["s3:prefix"] = "Asia/"
-		innerMapInValid := make(map[string]string)
-		innerMapInValid["s3:invalid"] = "Asia/"
-		conditions := make(map[string]map[string]string)
+	generateValidInvalidConditionKeys := func() map[string]map[string]set.StringSet {
+		innerMapValid := make(map[string]set.StringSet)
+		innerMapValid["s3:prefix"] = set.CreateStringSet("Asia/")
+		innerMapInValid := make(map[string]set.StringSet)
+		innerMapInValid["s3:invalid"] = set.CreateStringSet("Asia/")
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions["StringEquals"] = innerMapValid
 		conditions["StringEquals"] = innerMapInValid
 		return conditions
 	}
 
 	// List of Conditions used for test cases.
-	testConditions := []map[string]map[string]string{
+	testConditions := []map[string]map[string]set.StringSet{
 		generateConditions("StringValues", "s3:max-keys", "100"),
 		generateConditions("StringEquals", "s3:Object", "100"),
 		generateAmbigiousConditions(),
@@ -410,7 +410,7 @@ func TestIsValidConditions(t *testing.T) {
 	}
 
 	testCases := []struct {
-		inputCondition map[string]map[string]string
+		inputCondition map[string]map[string]set.StringSet
 		// expected result.
 		expectedErr error
 		// flag indicating whether test should pass.
@@ -474,20 +474,20 @@ func TestIsValidConditions(t *testing.T) {
 func TestCheckbucketPolicyResources(t *testing.T) {
 	// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
 	setValidPrefixActions := func(statements []policyStatement) []policyStatement {
-		statements[0].Actions = []string{"s3:DeleteObject", "s3:PutObject"}
+		statements[0].Actions = set.CreateStringSet([]string{"s3:DeleteObject", "s3:PutObject"}...)
 		return statements
 	}
 	// contracting policy statement with recursive resources.
 	// should result in ErrMalformedPolicy
 	setRecurseResource := func(statements []policyStatement) []policyStatement {
-		statements[0].Resources = []string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}
+		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}...)
 		return statements
 	}
 
 	// constructing policy statement with lexically close characters.
 	// should not result in ErrMalformedPolicy
 	setResourceLexical := func(statements []policyStatement) []policyStatement {
-		statements[0].Resources = []string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}
+		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}...)
 		return statements
 	}
 
@@ -575,7 +575,7 @@ func TestParseBucketPolicy(t *testing.T) {
 	// set Unsupported Actions.
 	setUnsupportedActions := func(statements []policyStatement) []policyStatement {
 		// "s3:DeleteEverything"" is an Unsupported Action.
-		statements[0].Actions = []string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}
+		statements[0].Actions = set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...)
 		return statements
 	}
 	// set unsupported Effect.
@@ -587,13 +587,13 @@ func TestParseBucketPolicy(t *testing.T) {
 	// set unsupported principals.
 	setUnsupportedPrincipals := func(statements []policyStatement) []policyStatement {
 		// "User1111"" is an Unsupported Principal.
-		statements[0].Principal.AWS = []string{"*", "User1111"}
+		statements[0].Principal.AWS = set.CreateStringSet([]string{"*", "User1111"}...)
 		return statements
 	}
 	// set unsupported Resources.
 	setUnsupportedResources := func(statements []policyStatement) []policyStatement {
 		// "s3:DeleteEverything"" is an Unsupported Action.
-		statements[0].Resources = []string{"my-resource"}
+		statements[0].Resources = set.CreateStringSet([]string{"my-resource"}...)
 		return statements
 	}
 	// List of bucketPolicy used for test cases.
@@ -652,13 +652,13 @@ func TestParseBucketPolicy(t *testing.T) {
 		{bucketAccesPolicies[4], bucketAccesPolicies[4], nil, true},
 		// Test case - 6.
 		// bucketPolicy statement contains unsupported action.
-		{bucketAccesPolicies[5], bucketAccesPolicies[5], fmt.Errorf("Unsupported action found: ‘s3:DeleteEverything’, please validate your policy document."), false},
+		{bucketAccesPolicies[5], bucketAccesPolicies[5], fmt.Errorf("Unsupported actions found: ‘set.StringSet{\"s3:DeleteEverything\":struct {}{}}’, please validate your policy document."), false},
 		// Test case - 7.
 		// bucketPolicy statement contains unsupported Effect.
 		{bucketAccesPolicies[6], bucketAccesPolicies[6], fmt.Errorf("Unsupported Effect found: ‘DontAllow’, please validate your policy document."), false},
 		// Test case - 8.
 		// bucketPolicy statement contains unsupported Principal.
-		{bucketAccesPolicies[7], bucketAccesPolicies[7], fmt.Errorf("Unsupported principal style found: ‘User1111’, please validate your policy document."), false},
+		{bucketAccesPolicies[7], bucketAccesPolicies[7], fmt.Errorf("Unsupported principals found: ‘set.StringSet{\"User1111\":struct {}{}}’, please validate your policy document."), false},
 		// Test case - 9.
 		// bucketPolicy statement contains unsupported Resource.
 		{bucketAccesPolicies[8], bucketAccesPolicies[8], fmt.Errorf("Unsupported resource style found: ‘my-resource’, please validate your policy document."), false},

cmd/bucket-policy.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"
@@ -116,9 +116,9 @@ func getOldBucketsConfigPath() (string, error) {
 	return path.Join(configPath, "buckets"), nil
 }
 
-// readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
-// if bucket policy is not found. This function also parses the bucket policy into an object.
-func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error) {
+// readBucketPolicyJSON - reads bucket policy for an input bucket, returns BucketPolicyNotFound
+// if bucket policy is not found.
+func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader io.Reader, err error) {
 	// Verify bucket is valid.
 	if !IsValidBucketName(bucket) {
 		return nil, BucketNameInvalid{Bucket: bucket}
@@ -139,9 +139,22 @@ func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error)
 		}
 		return nil, err
 	}
+
+	return &buffer, nil
+}
+
+// readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
+// if bucket policy is not found. This function also parses the bucket policy into an object.
+func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error) {
+	// Read bucket policy JSON.
+	bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI)
+	if err != nil {
+		return nil, err
+	}
+
 	// Parse the saved policy.
 	var policy = &bucketPolicy{}
-	err = parseBucketPolicy(&buffer, policy)
+	err = parseBucketPolicy(bucketPolicyReader, policy)
 	if err != nil {
 		return nil, err
 

cmd/build-constants.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,20 +14,20 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 // DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
 // set through ‘buildscripts/gen-ldflags.go’.
 var (
-	// minioGOPATH - GOPATH value at the time of build.
-	minioGOPATH = ""
+	// GOPATH - GOPATH value at the time of build.
+	GOPATH = ""
 
-	// minioVersion - version time.RFC3339.
-	minioVersion = "DEVELOPMENT.GOGET"
-	// minioReleaseTag - release tag in TAG.%Y-%m-%dT%H-%M-%SZ.
-	minioReleaseTag = "DEVELOPMENT.GOGET"
-	// minioCommitID - latest commit id.
-	minioCommitID = "DEVELOPMENT.GOGET"
-	// minioShortCommitID - first 12 characters from minioCommitID.
-	minioShortCommitID = minioCommitID[:12]
+	// Version - version time.RFC3339.
+	Version = "DEVELOPMENT.GOGET"
+	// ReleaseTag - release tag in TAG.%Y-%m-%dT%H-%M-%SZ.
+	ReleaseTag = "DEVELOPMENT.GOGET"
+	// CommitID - latest commit id.
+	CommitID = "DEVELOPMENT.GOGET"
+	// ShortCommitID - first 12 characters from CommitID.
+	ShortCommitID = CommitID[:12]
 )

cmd/certs.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"os"

cmd/certs_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"os"

cmd/checkport.go

@@ -14,11 +14,13 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"fmt"
 	"net"
+	"os"
+	"syscall"
 )
 
 // Make sure that none of the other processes are listening on the
@@ -35,7 +37,13 @@ func checkPortAvailability(port int) error {
 	for _, n := range network {
 		l, err := net.Listen(n, fmt.Sprintf(":%d", port))
 		if err != nil {
-			return err
+			if isAddrInUse(err) {
+				// Return error if another process is listening on the
+				// same port.
+				return err
+			}
+			// Ignore any other error (ex. EAFNOSUPPORT)
+			continue
 		}
 
 		// look for error so we don't have dangling connection
@@ -46,3 +54,18 @@ func checkPortAvailability(port int) error {
 
 	return nil
 }
+
+// Return true if err is "address already in use" error.
+// syscall.EADDRINUSE is available on all OSes.
+func isAddrInUse(err error) bool {
+	if opErr, ok := err.(*net.OpError); ok {
+		if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
+			if errno, ok := sysErr.Err.(syscall.Errno); ok {
+				if errno == syscall.EADDRINUSE {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

cmd/checkport_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"fmt"

cmd/commands.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "github.com/minio/cli"
 

cmd/config-migrate.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"errors"
@@ -36,6 +36,8 @@ func migrateConfig() {
 	migrateV4ToV5()
 	// Migrate version '5' to '6.
 	migrateV5ToV6()
+	// Migrate version '6' to '7'.
+	migrateV6ToV7()
 }
 
 // Version '1' is not supported anymore and deprecated, safe to delete.
@@ -151,6 +153,46 @@ func migrateV3ToV4() {
 	console.Println("Migration from version ‘" + cv3.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
 }
 
+// Version '4' to '5' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV4ToV5() {
+	cv4, err := loadConfigV4()
+	if err != nil && os.IsNotExist(err) {
+		return
+	}
+	fatalIf(err, "Unable to load config version ‘4’.")
+	if cv4.Version != "4" {
+		return
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &configV5{}
+	srvConfig.Version = "5"
+	srvConfig.Credential = cv4.Credential
+	srvConfig.Region = cv4.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv4.Logger.Console
+	srvConfig.Logger.File = cv4.Logger.File
+	srvConfig.Logger.Syslog = cv4.Logger.Syslog
+	srvConfig.Logger.AMQP.Enable = false
+	srvConfig.Logger.ElasticSearch.Enable = false
+	srvConfig.Logger.Redis.Enable = false
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err, "Unable to initialize the quick config.")
+	configFile, err := getConfigFile()
+	fatalIf(err, "Unable to get config file.")
+
+	err = qc.Save(configFile)
+	fatalIf(err, "Failed to migrate config from ‘"+cv4.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
+
+	console.Println("Migration from version ‘" + cv4.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}
+
 // Version '5' to '6' migrates config, removes previous fields related
 // to backend types and server address. This change further simplifies
 // the config for future additions.
@@ -165,8 +207,8 @@ func migrateV5ToV6() {
 	}
 
 	// Save only the new fields, ignore the rest.
-	srvConfig := &serverConfigV6{}
-	srvConfig.Version = globalMinioConfigVersion
+	srvConfig := &configV6{}
+	srvConfig.Version = "6"
 	srvConfig.Credential = cv5.Credential
 	srvConfig.Region = cv5.Region
 	if srvConfig.Region == "" {
@@ -176,6 +218,7 @@ func migrateV5ToV6() {
 	srvConfig.Logger.Console = cv5.Logger.Console
 	srvConfig.Logger.File = cv5.Logger.File
 	srvConfig.Logger.Syslog = cv5.Logger.Syslog
+
 	srvConfig.Notify.AMQP = map[string]amqpNotify{
 		"1": {
 			Enable:      cv5.Logger.AMQP.Enable,
@@ -217,34 +260,49 @@ func migrateV5ToV6() {
 	console.Println("Migration from version ‘" + cv5.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
 }
 
-// Version '4' to '5' migrates config, removes previous fields related
+// Version '6' to '7' migrates config, removes previous fields related
 // to backend types and server address. This change further simplifies
 // the config for future additions.
-func migrateV4ToV5() {
-	cv4, err := loadConfigV4()
+func migrateV6ToV7() {
+	cv6, err := loadConfigV6()
 	if err != nil && os.IsNotExist(err) {
 		return
 	}
-	fatalIf(err, "Unable to load config version ‘4’.")
-	if cv4.Version != "4" {
+	fatalIf(err, "Unable to load config version ‘6’.")
+	if cv6.Version != "6" {
 		return
 	}
 
 	// Save only the new fields, ignore the rest.
-	srvConfig := &configV5{}
+	srvConfig := &serverConfigV7{}
 	srvConfig.Version = globalMinioConfigVersion
-	srvConfig.Credential = cv4.Credential
-	srvConfig.Region = cv4.Region
+	srvConfig.Credential = cv6.Credential
+	srvConfig.Region = cv6.Region
 	if srvConfig.Region == "" {
 		// Region needs to be set for AWS Signature Version 4.
 		srvConfig.Region = "us-east-1"
 	}
-	srvConfig.Logger.Console = cv4.Logger.Console
-	srvConfig.Logger.File = cv4.Logger.File
-	srvConfig.Logger.Syslog = cv4.Logger.Syslog
-	srvConfig.Logger.AMQP.Enable = false
-	srvConfig.Logger.ElasticSearch.Enable = false
-	srvConfig.Logger.Redis.Enable = false
+	srvConfig.Logger.Console = cv6.Logger.Console
+	srvConfig.Logger.File = cv6.Logger.File
+	srvConfig.Logger.Syslog = cv6.Logger.Syslog
+	srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+	srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+	srvConfig.Notify.Redis = make(map[string]redisNotify)
+	if len(cv6.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		srvConfig.Notify.AMQP = cv6.Notify.AMQP
+	}
+	if len(cv6.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv6.Notify.ElasticSearch
+	}
+	if len(cv6.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis["1"] = redisNotify{}
+	} else {
+		srvConfig.Notify.Redis = cv6.Notify.Redis
+	}
 
 	qc, err := quick.New(srvConfig)
 	fatalIf(err, "Unable to initialize the quick config.")
@@ -252,7 +310,7 @@ func migrateV4ToV5() {
 	fatalIf(err, "Unable to get config file.")
 
 	err = qc.Save(configFile)
-	fatalIf(err, "Failed to migrate config from ‘"+cv4.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
+	fatalIf(err, "Failed to migrate config from ‘"+cv6.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
 
-	console.Println("Migration from version ‘" + cv4.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+	console.Println("Migration from version ‘" + cv6.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
 }

cmd/config-old.go

@@ -1,4 +1,4 @@
-package main
+package cmd
 
 import (
 	"os"
@@ -274,3 +274,39 @@ func loadConfigV5() (*configV5, error) {
 	}
 	return c, nil
 }
+
+// configV6 server configuration version '6'.
+type configV6 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger logger `json:"logger"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// loadConfigV6 load config version '6'.
+func loadConfigV6() (*configV6, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV6{}
+	c.Version = "6"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

cmd/config-v7.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"os"
@@ -23,8 +23,8 @@ import (
 	"github.com/minio/minio/pkg/quick"
 )
 
-// serverConfigV6 server configuration version '5'.
-type serverConfigV6 struct {
+// serverConfigV7 server configuration version '7'.
+type serverConfigV7 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
@@ -45,7 +45,7 @@ type serverConfigV6 struct {
 func initConfig() error {
 	if !isConfigFileExists() {
 		// Initialize server config.
-		srvCfg := &serverConfigV6{}
+		srvCfg := &serverConfigV7{}
 		srvCfg.Version = globalMinioConfigVersion
 		srvCfg.Region = "us-east-1"
 		srvCfg.Credential = mustGenAccessKeys()
@@ -55,6 +55,14 @@ func initConfig() error {
 			Enable: true,
 			Level:  "fatal",
 		}
+
+		// Make sure to initialize notification configs.
+		srvCfg.Notify.AMQP = make(map[string]amqpNotify)
+		srvCfg.Notify.AMQP["1"] = amqpNotify{}
+		srvCfg.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvCfg.Notify.ElasticSearch["1"] = elasticSearchNotify{}
+		srvCfg.Notify.Redis = make(map[string]redisNotify)
+		srvCfg.Notify.Redis["1"] = redisNotify{}
 		srvCfg.rwMutex = &sync.RWMutex{}
 
 		// Create config path.
@@ -76,7 +84,7 @@ func initConfig() error {
 	if _, err = os.Stat(configFile); err != nil {
 		return err
 	}
-	srvCfg := &serverConfigV6{}
+	srvCfg := &serverConfigV7{}
 	srvCfg.Version = globalMinioConfigVersion
 	srvCfg.rwMutex = &sync.RWMutex{}
 	qc, err := quick.New(srvCfg)
@@ -95,10 +103,10 @@ func initConfig() error {
 }
 
 // serverConfig server config.
-var serverConfig *serverConfigV6
+var serverConfig *serverConfigV7
 
 // GetVersion get current config version.
-func (s serverConfigV6) GetVersion() string {
+func (s serverConfigV7) GetVersion() string {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Version
@@ -106,135 +114,135 @@ func (s serverConfigV6) GetVersion() string {
 
 /// Logger related.
 
-func (s *serverConfigV6) SetAMQPNotifyByID(accountID string, amqpn amqpNotify) {
+func (s *serverConfigV7) SetAMQPNotifyByID(accountID string, amqpn amqpNotify) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Notify.AMQP[accountID] = amqpn
 }
 
-func (s serverConfigV6) GetAMQP() map[string]amqpNotify {
+func (s serverConfigV7) GetAMQP() map[string]amqpNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.AMQP
 }
 
 // GetAMQPNotify get current AMQP logger.
-func (s serverConfigV6) GetAMQPNotifyByID(accountID string) amqpNotify {
+func (s serverConfigV7) GetAMQPNotifyByID(accountID string) amqpNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.AMQP[accountID]
 }
 
-func (s *serverConfigV6) SetElasticSearchNotifyByID(accountID string, esNotify elasticSearchNotify) {
+func (s *serverConfigV7) SetElasticSearchNotifyByID(accountID string, esNotify elasticSearchNotify) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Notify.ElasticSearch[accountID] = esNotify
 }
 
-func (s serverConfigV6) GetElasticSearch() map[string]elasticSearchNotify {
+func (s serverConfigV7) GetElasticSearch() map[string]elasticSearchNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.ElasticSearch
 }
 
 // GetElasticSearchNotify get current ElasicSearch logger.
-func (s serverConfigV6) GetElasticSearchNotifyByID(accountID string) elasticSearchNotify {
+func (s serverConfigV7) GetElasticSearchNotifyByID(accountID string) elasticSearchNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.ElasticSearch[accountID]
 }
 
-func (s *serverConfigV6) SetRedisNotifyByID(accountID string, rNotify redisNotify) {
+func (s *serverConfigV7) SetRedisNotifyByID(accountID string, rNotify redisNotify) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Notify.Redis[accountID] = rNotify
 }
 
-func (s serverConfigV6) GetRedis() map[string]redisNotify {
+func (s serverConfigV7) GetRedis() map[string]redisNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.Redis
 }
 
 // GetRedisNotify get current Redis logger.
-func (s serverConfigV6) GetRedisNotifyByID(accountID string) redisNotify {
+func (s serverConfigV7) GetRedisNotifyByID(accountID string) redisNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.Redis[accountID]
 }
 
 // SetFileLogger set new file logger.
-func (s *serverConfigV6) SetFileLogger(flogger fileLogger) {
+func (s *serverConfigV7) SetFileLogger(flogger fileLogger) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Logger.File = flogger
 }
 
 // GetFileLogger get current file logger.
-func (s serverConfigV6) GetFileLogger() fileLogger {
+func (s serverConfigV7) GetFileLogger() fileLogger {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Logger.File
 }
 
 // SetConsoleLogger set new console logger.
-func (s *serverConfigV6) SetConsoleLogger(clogger consoleLogger) {
+func (s *serverConfigV7) SetConsoleLogger(clogger consoleLogger) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Logger.Console = clogger
 }
 
 // GetConsoleLogger get current console logger.
-func (s serverConfigV6) GetConsoleLogger() consoleLogger {
+func (s serverConfigV7) GetConsoleLogger() consoleLogger {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Logger.Console
 }
 
 // SetSyslogLogger set new syslog logger.
-func (s *serverConfigV6) SetSyslogLogger(slogger syslogLogger) {
+func (s *serverConfigV7) SetSyslogLogger(slogger syslogLogger) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Logger.Syslog = slogger
 }
 
 // GetSyslogLogger get current syslog logger.
-func (s *serverConfigV6) GetSyslogLogger() syslogLogger {
+func (s *serverConfigV7) GetSyslogLogger() syslogLogger {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Logger.Syslog
 }
 
 // SetRegion set new region.
-func (s *serverConfigV6) SetRegion(region string) {
+func (s *serverConfigV7) SetRegion(region string) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Region = region
 }
 
 // GetRegion get current region.
-func (s serverConfigV6) GetRegion() string {
+func (s serverConfigV7) GetRegion() string {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Region
 }
 
 // SetCredentials set new credentials.
-func (s *serverConfigV6) SetCredential(creds credential) {
+func (s *serverConfigV7) SetCredential(creds credential) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Credential = creds
 }
 
 // GetCredentials get current credentials.
-func (s serverConfigV6) GetCredential() credential {
+func (s serverConfigV7) GetCredential() credential {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Credential
 }
 
 // Save config.
-func (s serverConfigV6) Save() error {
+func (s serverConfigV7) Save() error {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 

cmd/config-v7_test.go

@@ -0,0 +1,107 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestServerConfig(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	if serverConfig.GetRegion() != "us-east-1" {
+		t.Errorf("Expecting region `us-east-1` found %s", serverConfig.GetRegion())
+	}
+
+	// Set new region and verify.
+	serverConfig.SetRegion("us-west-1")
+	if serverConfig.GetRegion() != "us-west-1" {
+		t.Errorf("Expecting region `us-west-1` found %s", serverConfig.GetRegion())
+	}
+
+	// Set new amqp notification id.
+	serverConfig.SetAMQPNotifyByID("2", amqpNotify{})
+	savedNotifyCfg1 := serverConfig.GetAMQPNotifyByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg1, amqpNotify{}) {
+		t.Errorf("Expecting AMQP config %#v found %#v", amqpNotify{}, savedNotifyCfg1)
+	}
+
+	// Set new elastic search notification id.
+	serverConfig.SetElasticSearchNotifyByID("2", elasticSearchNotify{})
+	savedNotifyCfg2 := serverConfig.GetElasticSearchNotifyByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg2, elasticSearchNotify{}) {
+		t.Errorf("Expecting Elasticsearch config %#v found %#v", elasticSearchNotify{}, savedNotifyCfg2)
+	}
+
+	// Set new redis notification id.
+	serverConfig.SetRedisNotifyByID("2", redisNotify{})
+	savedNotifyCfg3 := serverConfig.GetRedisNotifyByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg3, redisNotify{}) {
+		t.Errorf("Expecting Redis config %#v found %#v", redisNotify{}, savedNotifyCfg3)
+	}
+
+	// Set new console logger.
+	serverConfig.SetConsoleLogger(consoleLogger{
+		Enable: true,
+	})
+	consoleCfg := serverConfig.GetConsoleLogger()
+	if !reflect.DeepEqual(consoleCfg, consoleLogger{Enable: true}) {
+		t.Errorf("Expecting console logger config %#v found %#v", consoleLogger{Enable: true}, consoleCfg)
+	}
+
+	// Set new file logger.
+	serverConfig.SetFileLogger(fileLogger{
+		Enable: true,
+	})
+	fileCfg := serverConfig.GetFileLogger()
+	if !reflect.DeepEqual(fileCfg, fileLogger{Enable: true}) {
+		t.Errorf("Expecting file logger config %#v found %#v", fileLogger{Enable: true}, consoleCfg)
+	}
+
+	// Set new syslog logger.
+	serverConfig.SetSyslogLogger(syslogLogger{
+		Enable: true,
+	})
+	sysLogCfg := serverConfig.GetSyslogLogger()
+	if !reflect.DeepEqual(sysLogCfg, syslogLogger{Enable: true}) {
+		t.Errorf("Expecting syslog logger config %#v found %#v", syslogLogger{Enable: true}, sysLogCfg)
+	}
+
+	// Match version.
+	if serverConfig.GetVersion() != globalMinioConfigVersion {
+		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), globalMinioConfigVersion)
+	}
+
+	// Attempt to save.
+	if err := serverConfig.Save(); err != nil {
+		t.Fatalf("Unable to save updated config file %s", err)
+	}
+
+	// Do this only once here.
+	setGlobalConfigPath(rootPath)
+
+	// Initialize server config.
+	if err := initConfig(); err != nil {
+		t.Fatalf("Unable to initialize from updated config file %s", err)
+	}
+}

cmd/config.go

@@ -14,25 +14,32 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"os"
 	"path/filepath"
+	"sync"
 
 	"github.com/minio/go-homedir"
 )
 
 // configPath for custom config path only for testing purposes
 var customConfigPath string
+var configMu sync.Mutex
 
 // Sets a new config path.
 func setGlobalConfigPath(configPath string) {
+	configMu.Lock()
+	defer configMu.Unlock()
 	customConfigPath = configPath
 }
 
 // getConfigPath get server config path
 func getConfigPath() (string, error) {
+	configMu.Lock()
+	defer configMu.Unlock()
+
 	if customConfigPath != "" {
 		return customConfigPath, nil
 	}

cmd/control-heal-main.go

@@ -0,0 +1,120 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"net/rpc"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/minio/cli"
+)
+
+var healCmd = cli.Command{
+	Name:   "heal",
+	Usage:  "To heal objects.",
+	Action: healControl,
+	CustomHelpTemplate: `NAME:
+  minio control {{.Name}} - {{.Usage}}
+
+USAGE:
+  minio control {{.Name}}
+
+EAMPLES:
+  1. Heal an object.
+     $ minio control {{.Name}} http://localhost:9000/songs/classical/western/piano.mp3
+
+  2. Heal all objects in a bucket recursively.
+     $ minio control {{.Name}}  http://localhost:9000/songs
+
+  3. Heall all objects with a given prefix recursively.
+     $ minio control {{.Name}} http://localhost:9000/songs/classical/
+`,
+}
+
+// "minio control heal" entry point.
+func healControl(ctx *cli.Context) {
+	// Parse bucket and object from url.URL.Path
+	parseBucketObject := func(path string) (bucketName string, objectName string) {
+		splits := strings.SplitN(path, string(slashSeparator), 3)
+		switch len(splits) {
+		case 0, 1:
+			bucketName = ""
+			objectName = ""
+		case 2:
+			bucketName = splits[1]
+			objectName = ""
+		case 3:
+			bucketName = splits[1]
+			objectName = splits[2]
+
+		}
+		return bucketName, objectName
+	}
+
+	if len(ctx.Args()) != 1 {
+		cli.ShowCommandHelpAndExit(ctx, "heal", 1)
+	}
+
+	parsedURL, err := url.Parse(ctx.Args()[0])
+	fatalIf(err, "Unable to parse URL")
+
+	bucketName, objectName := parseBucketObject(parsedURL.Path)
+	if bucketName == "" {
+		cli.ShowCommandHelpAndExit(ctx, "heal", 1)
+	}
+
+	client, err := rpc.DialHTTPPath("tcp", parsedURL.Host, path.Join(reservedBucket, controlPath))
+	fatalIf(err, "Unable to connect to %s", parsedURL.Host)
+
+	// If object does not have trailing "/" then it's an object, hence heal it.
+	if objectName != "" && !strings.HasSuffix(objectName, slashSeparator) {
+		fmt.Printf("Healing : /%s/%s", bucketName, objectName)
+		args := &HealObjectArgs{bucketName, objectName}
+		reply := &HealObjectReply{}
+		err = client.Call("Control.HealObject", args, reply)
+		fatalIf(err, "RPC Control.HealObject call failed")
+		fmt.Println()
+		return
+	}
+
+	// Recursively list and heal the objects.
+	prefix := objectName
+	marker := ""
+	for {
+		args := HealListArgs{bucketName, prefix, marker, "", 1000}
+		reply := &HealListReply{}
+		err = client.Call("Control.ListObjectsHeal", args, reply)
+		fatalIf(err, "RPC Heal.ListObjects call failed")
+
+		// Heal the objects returned in the ListObjects reply.
+		for _, obj := range reply.Objects {
+			fmt.Printf("Healing : /%s/%s", bucketName, obj)
+			reply := &HealObjectReply{}
+			err = client.Call("Control.HealObject", HealObjectArgs{bucketName, obj}, reply)
+			fatalIf(err, "RPC Heal.HealObject call failed")
+			fmt.Println()
+		}
+		if !reply.IsTruncated {
+			// End of listing.
+			break
+		}
+		marker = reply.NextMarker
+	}
+}

cmd/control-main.go

@@ -0,0 +1,52 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "github.com/minio/cli"
+
+// "minio control" command.
+var controlCmd = cli.Command{
+	Name:   "control",
+	Usage:  "Control and manage minio server.",
+	Action: mainControl,
+	Subcommands: []cli.Command{
+		healCmd,
+		shutdownCmd,
+	},
+	CustomHelpTemplate: `NAME:
+   {{.Name}} - {{.Usage}}
+
+USAGE:
+   {{.Name}} [FLAGS] COMMAND
+
+FLAGS:
+  {{range .Flags}}{{.}}
+  {{end}}
+COMMANDS:
+   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
+   {{end}}
+`,
+}
+
+func mainControl(ctx *cli.Context) {
+	if ctx.Args().First() != "" { // command help.
+		cli.ShowCommandHelp(ctx, ctx.Args().First())
+	} else {
+		// command with Subcommands is an App.
+		cli.ShowAppHelp(ctx)
+	}
+}

cmd/control-shutdown-main.go

@@ -0,0 +1,68 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/rpc"
+	"net/url"
+	"path"
+
+	"github.com/minio/cli"
+)
+
+var shutdownCmd = cli.Command{
+	Name:   "shutdown",
+	Usage:  "Shutdown or restart the server.",
+	Action: shutdownControl,
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "restart",
+			Usage: "Restart the server.",
+		},
+	},
+	CustomHelpTemplate: `NAME:
+  minio control {{.Name}} - {{.Usage}}
+
+USAGE:
+  minio control {{.Name}} http://localhost:9000/
+
+EAMPLES:
+  1. Shutdown the server:
+    $ minio control shutdown http://localhost:9000/
+
+  2. Reboot the server:
+    $ minio control shutdown --restart http://localhost:9000/
+`,
+}
+
+// "minio control shutdown" entry point.
+func shutdownControl(c *cli.Context) {
+	if len(c.Args()) != 1 {
+		cli.ShowCommandHelpAndExit(c, "shutdown", 1)
+	}
+
+	parsedURL, err := url.ParseRequestURI(c.Args()[0])
+	fatalIf(err, "Unable to parse URL")
+
+	client, err := rpc.DialHTTPPath("tcp", parsedURL.Host, path.Join(reservedBucket, controlPath))
+	fatalIf(err, "Unable to connect to %s", parsedURL.Host)
+
+	args := &ShutdownArgs{Reboot: c.Bool("restart")}
+	reply := &ShutdownReply{}
+	err = client.Call("Control.Shutdown", args, reply)
+	fatalIf(err, "RPC Control.Shutdown call failed")
+}

cmd/controller-handlers.go

@@ -0,0 +1,88 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// HealListArgs - argument for ListObjects RPC.
+type HealListArgs struct {
+	Bucket    string
+	Prefix    string
+	Marker    string
+	Delimiter string
+	MaxKeys   int
+}
+
+// HealListReply - reply by ListObjects RPC.
+type HealListReply struct {
+	IsTruncated bool
+	NextMarker  string
+	Objects     []string
+}
+
+// ListObjects - list all objects that needs healing.
+func (c *controllerAPIHandlers) ListObjectsHeal(arg *HealListArgs, reply *HealListReply) error {
+	objAPI := c.ObjectAPI
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+	info, err := objAPI.ListObjectsHeal(arg.Bucket, arg.Prefix, arg.Marker, arg.Delimiter, arg.MaxKeys)
+	if err != nil {
+		return err
+	}
+	reply.IsTruncated = info.IsTruncated
+	reply.NextMarker = info.NextMarker
+	for _, obj := range info.Objects {
+		reply.Objects = append(reply.Objects, obj.Name)
+	}
+	return nil
+}
+
+// HealObjectArgs - argument for HealObject RPC.
+type HealObjectArgs struct {
+	Bucket string
+	Object string
+}
+
+// HealObjectReply - reply by HealObject RPC.
+type HealObjectReply struct{}
+
+// HealObject - heal the object.
+func (c *controllerAPIHandlers) HealObject(arg *HealObjectArgs, reply *HealObjectReply) error {
+	objAPI := c.ObjectAPI
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+	return objAPI.HealObject(arg.Bucket, arg.Object)
+}
+
+// ShutdownArgs - argument for Shutdown RPC.
+type ShutdownArgs struct {
+	Reboot bool
+}
+
+// ShutdownReply - reply by Shutdown RPC.
+type ShutdownReply struct{}
+
+// Shutdown - Shutdown the server.
+
+func (c *controllerAPIHandlers) Shutdown(arg *ShutdownArgs, reply *ShutdownReply) error {
+	if arg.Reboot {
+		globalShutdownSignalCh <- shutdownRestart
+	} else {
+		globalShutdownSignalCh <- shutdownHalt
+	}
+	return nil
+}

cmd/controller-router.go

@@ -0,0 +1,42 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/rpc"
+
+	router "github.com/gorilla/mux"
+)
+
+// Routes paths for "minio control" commands.
+const (
+	controlPath = "/controller"
+)
+
+// Register control RPC handlers.
+func registerControlRPCRouter(mux *router.Router, ctrlHandlers *controllerAPIHandlers) {
+	ctrlRPCServer := rpc.NewServer()
+	ctrlRPCServer.RegisterName("Control", ctrlHandlers)
+
+	ctrlRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter()
+	ctrlRouter.Path(controlPath).Handler(ctrlRPCServer)
+}
+
+// Handler for object healing.
+type controllerAPIHandlers struct {
+	ObjectAPI ObjectLayer
+}

cmd/crossdomain-xml-handler.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "net/http"
 

cmd/damerau-levenshtein.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"math"

cmd/damerau-levenshtein_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"math"

cmd/erasure-createfile.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"encoding/hex"

cmd/erasure-createfile_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/erasure-healfile.go

@@ -0,0 +1,84 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "encoding/hex"
+
+// Heals the erasure coded file. reedsolomon.Reconstruct() is used to reconstruct the missing parts.
+func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string, size int64, blockSize int64, dataBlocks int, parityBlocks int, algo string) (checkSums []string, err error) {
+	var offset int64
+	remainingSize := size
+
+	// Hash for bitrot protection.
+	hashWriters := newHashWriters(len(outDatedDisks), bitRotAlgo)
+
+	for remainingSize > 0 {
+		curBlockSize := blockSize
+		if remainingSize < curBlockSize {
+			curBlockSize = remainingSize
+		}
+
+		// Calculate the block size that needs to be read from each disk.
+		curEncBlockSize := getChunkSize(curBlockSize, dataBlocks)
+
+		// Memory for reading data from disks and reconstructing missing data using erasure coding.
+		enBlocks := make([][]byte, len(latestDisks))
+
+		// Read data from the latest disks.
+		// FIXME: no need to read from all the disks. dataBlocks+1 is enough.
+		for index, disk := range latestDisks {
+			if disk == nil {
+				continue
+			}
+			enBlocks[index] = make([]byte, curEncBlockSize)
+			_, err := disk.ReadFile(volume, path, offset, enBlocks[index])
+			if err != nil {
+				enBlocks[index] = nil
+			}
+		}
+
+		// Reconstruct missing data.
+		err := decodeData(enBlocks, dataBlocks, parityBlocks)
+		if err != nil {
+			return nil, err
+		}
+
+		// Write to the healPath file.
+		for index, disk := range outDatedDisks {
+			if disk == nil {
+				continue
+			}
+			err := disk.AppendFile(healBucket, healPath, enBlocks[index])
+			if err != nil {
+				return nil, err
+			}
+			hashWriters[index].Write(enBlocks[index])
+		}
+		remainingSize -= curBlockSize
+		offset += curEncBlockSize
+	}
+
+	// Checksums for the bit rot.
+	checkSums = make([]string, len(outDatedDisks))
+	for index, disk := range outDatedDisks {
+		if disk == nil {
+			continue
+		}
+		checkSums[index] = hex.EncodeToString(hashWriters[index].Sum(nil))
+	}
+	return checkSums, nil
+}

cmd/erasure-healfile_test.go

@@ -0,0 +1,123 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/rand"
+	"os"
+	"path"
+	"testing"
+)
+
+// Test erasureHealFile()
+func TestErasureHealFile(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(blockSizeV1)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 1MB with random data.
+	data := make([]byte, 1*1024*1024)
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Create a test file.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	latest := make([]StorageAPI, len(disks))   // Slice of latest disks
+	outDated := make([]StorageAPI, len(disks)) // Slice of outdated disks
+
+	// Test case when one part needs to be healed.
+	dataPath := path.Join(setup.diskPaths[0], "testbucket", "testobject1")
+	err = os.Remove(dataPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	copy(latest, disks)
+	latest[0] = nil
+	outDated[0] = disks[0]
+	healCheckSums, err := erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*1024*1024, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	// Checksum of the healed file should match.
+	if checkSums[0] != healCheckSums[0] {
+		t.Error("Healing failed, data does not match.")
+	}
+
+	// Test case when parityBlocks number of disks need to be healed.
+	// Should succeed.
+	copy(latest, disks)
+	for index := 0; index < parityBlocks; index++ {
+		dataPath := path.Join(setup.diskPaths[index], "testbucket", "testobject1")
+		err = os.Remove(dataPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		latest[index] = nil
+		outDated[index] = disks[index]
+	}
+
+	healCheckSums, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*1024*1024, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Checksums of the healed files should match.
+	for index := 0; index < parityBlocks; index++ {
+		if checkSums[index] != healCheckSums[index] {
+			t.Error("Healing failed, data does not match.")
+		}
+	}
+	for index := dataBlocks; index < len(disks); index++ {
+		if healCheckSums[index] != "" {
+			t.Errorf("expected healCheckSums[%d] to be empty", index)
+		}
+	}
+
+	// Test case when parityBlocks+1 number of disks need to be healed.
+	// Should fail.
+	copy(latest, disks)
+	for index := 0; index < parityBlocks+1; index++ {
+		dataPath := path.Join(setup.diskPaths[index], "testbucket", "testobject1")
+		err = os.Remove(dataPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		latest[index] = nil
+		outDated[index] = disks[index]
+	}
+	healCheckSums, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*1024*1024, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	if err == nil {
+		t.Error("Expected erasureHealFile() to fail when the number of available disks <= parityBlocks")
+	}
+}

cmd/erasure-readfile.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"encoding/hex"

cmd/erasure-readfile_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"
@@ -112,7 +112,7 @@ func testGetReadDisks(t *testing.T, xl xlObjects) {
 			t.Errorf("test-case %d - expected nextIndex: %d, got : %d", i+1, test.nextIndex, nextIndex)
 			continue
 		}
-		if reflect.DeepEqual(test.retDisks, disks) == false {
+		if !reflect.DeepEqual(test.retDisks, disks) {
 			t.Errorf("test-case %d : incorrect disks returned. expected %+v, got %+v", i+1, test.retDisks, disks)
 			continue
 		}

cmd/erasure-utils.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/erasure-utils_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/erasure_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/event-notifier.go

@@ -14,12 +14,11 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"
 	"encoding/xml"
-	"errors"
 	"fmt"
 	"net/url"
 	"path"
@@ -112,7 +111,7 @@ func (en *eventNotifier) SetSNSTarget(snsARN string, listenerCh chan []Notificat
 	en.rwMutex.Lock()
 	defer en.rwMutex.Unlock()
 	if listenerCh == nil {
-		return errors.New("invalid argument")
+		return errInvalidArgument
 	}
 	en.snsTargets[snsARN] = append(en.snsTargets[snsARN], listenerCh)
 	return nil
@@ -161,7 +160,7 @@ func (en *eventNotifier) SetBucketNotificationConfig(bucket string, notification
 	en.rwMutex.Lock()
 	defer en.rwMutex.Unlock()
 	if notificationCfg == nil {
-		return errors.New("invalid argument")
+		return errInvalidArgument
 	}
 	en.notificationConfigs[bucket] = notificationCfg
 	return nil
@@ -178,8 +177,11 @@ func eventNotify(event eventData) {
 	//  - s3:ObjectCreated:CompleteMultipartUpload
 	//  - s3:ObjectRemoved:Delete
 
-	nConfig := eventN.GetBucketNotificationConfig(event.Bucket)
+	nConfig := globalEventNotifier.GetBucketNotificationConfig(event.Bucket)
 	// No bucket notifications enabled, drop the event notification.
+	if nConfig == nil {
+		return
+	}
 	if len(nConfig.QueueConfigs) == 0 && len(nConfig.TopicConfigs) == 0 && len(nConfig.LambdaConfigs) == 0 {
 		return
 	}
@@ -198,7 +200,7 @@ func eventNotify(event eventData) {
 		eventMatch := eventMatch(eventType, qConfig.Events)
 		ruleMatch := filterRuleMatch(objectName, qConfig.Filter.Key.FilterRules)
 		if eventMatch && ruleMatch {
-			targetLog := eventN.GetQueueTarget(qConfig.QueueARN)
+			targetLog := globalEventNotifier.GetQueueTarget(qConfig.QueueARN)
 			if targetLog != nil {
 				targetLog.WithFields(logrus.Fields{
 					"Records": notificationEvent,
@@ -211,7 +213,7 @@ func eventNotify(event eventData) {
 		ruleMatch := filterRuleMatch(objectName, topicConfig.Filter.Key.FilterRules)
 		eventMatch := eventMatch(eventType, topicConfig.Events)
 		if eventMatch && ruleMatch {
-			targetListeners := eventN.GetSNSTarget(topicConfig.TopicARN)
+			targetListeners := globalEventNotifier.GetSNSTarget(topicConfig.TopicARN)
 			for _, listener := range targetListeners {
 				listener <- notificationEvent
 			}
@@ -352,7 +354,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 }
 
 // Global instance of event notification queue.
-var eventN *eventNotifier
+var globalEventNotifier *eventNotifier
 
 // Initialize event notifier.
 func initEventNotifier(objAPI ObjectLayer) error {
@@ -373,7 +375,7 @@ func initEventNotifier(objAPI ObjectLayer) error {
 	}
 
 	// Inititalize event notifier queue.
-	eventN = &eventNotifier{
+	globalEventNotifier = &eventNotifier{
 		rwMutex:             &sync.RWMutex{},
 		notificationConfigs: configs,
 		queueTargets:        queueTargets,

cmd/event-notifier_test.go

@@ -0,0 +1,133 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"reflect"
+	"testing"
+)
+
+// Tests event notify.
+func TestEventNotify(t *testing.T) {
+	ExecObjectLayerTest(t, testEventNotify)
+}
+
+func testEventNotify(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	bucketName := getRandomBucketName()
+
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	initEventNotifier(obj)
+
+	// Notify object created event.
+	eventNotify(eventData{
+		Type:   ObjectCreatedPost,
+		Bucket: bucketName,
+		ObjInfo: ObjectInfo{
+			Bucket: bucketName,
+			Name:   "object1",
+		},
+		ReqParams: map[string]string{
+			"sourceIPAddress": "localhost:1337",
+		},
+	})
+
+	if err := globalEventNotifier.SetBucketNotificationConfig(bucketName, nil); err != errInvalidArgument {
+		t.Errorf("Expected error %s, got %s", errInvalidArgument, err)
+	}
+
+	if err := globalEventNotifier.SetBucketNotificationConfig(bucketName, &notificationConfig{}); err != nil {
+		t.Errorf("Expected error to be nil, got %s", err)
+	}
+
+	if !globalEventNotifier.IsBucketNotificationSet(bucketName) {
+		t.Errorf("Notification expected to be set, but notification not set.")
+	}
+
+	nConfig := globalEventNotifier.GetBucketNotificationConfig(bucketName)
+	if !reflect.DeepEqual(nConfig, &notificationConfig{}) {
+		t.Errorf("Mismatching notification configs.")
+	}
+
+	// Notify object created event.
+	eventNotify(eventData{
+		Type:   ObjectCreatedPost,
+		Bucket: bucketName,
+		ObjInfo: ObjectInfo{
+			Bucket: bucketName,
+			Name:   "object1",
+		},
+		ReqParams: map[string]string{
+			"sourceIPAddress": "localhost:1337",
+		},
+	})
+}
+
+// Tests various forms of inititalization of event notifier.
+func TestInitEventNotifier(t *testing.T) {
+	fs, disk, err := getSingleNodeObjectLayer()
+	if err != nil {
+		t.Fatal("Unable to initialize FS backend.", err)
+	}
+	xl, disks, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal("Unable to initialize XL backend.", err)
+	}
+
+	disks = append(disks, disk)
+	for _, d := range disks {
+		defer removeAll(d)
+	}
+
+	// Collection of test cases for inititalizing event notifier.
+	testCases := []struct {
+		objAPI  ObjectLayer
+		configs map[string]*notificationConfig
+		err     error
+	}{
+		// Test 1 - invalid arguments.
+		{
+			objAPI: nil,
+			err:    errInvalidArgument,
+		},
+		// Test 2 - valid FS object layer but no bucket notifications.
+		{
+			objAPI: fs,
+			err:    nil,
+		},
+		// Test 3 - valid XL object layer but no bucket notifications.
+		{
+			objAPI: xl,
+			err:    nil,
+		},
+	}
+
+	// Validate if event notifier is properly initialized.
+	for i, testCase := range testCases {
+		err = initEventNotifier(testCase.objAPI)
+		if err != testCase.err {
+			t.Errorf("Test %d: Expected %s, but got: %s", i+1, testCase.err, err)
+		}
+	}
+}

cmd/format-config-v1.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"encoding/json"

cmd/format-config-v1_test.go

@@ -0,0 +1,943 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+)
+
+// generates a valid format.json for XL backend.
+func genFormatXLValid() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	return formatConfigs
+}
+
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidVersion() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Corrupt version numbers.
+	formatConfigs[0].Version = "2"
+	formatConfigs[3].Version = "-1"
+	return formatConfigs
+}
+
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidFormat() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Corrupt version numbers.
+	formatConfigs[0].Format = "lx"
+	formatConfigs[3].Format = "lx"
+	return formatConfigs
+}
+
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidXLVersion() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Corrupt version numbers.
+	formatConfigs[0].XL.Version = "10"
+	formatConfigs[3].XL.Version = "-1"
+	return formatConfigs
+}
+
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidJBODCount() []*formatConfigV1 {
+	jbod := make([]string, 7)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	return formatConfigs
+}
+
+// generates a invalid format.json JBOD for XL backend.
+func genFormatXLInvalidJBOD() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	// Corrupt JBOD entries on disk 6 and disk 8.
+	formatConfigs[5].XL.JBOD = jbod
+	formatConfigs[7].XL.JBOD = jbod
+	return formatConfigs
+}
+
+// generates a invalid format.json Disk UUID for XL backend.
+func genFormatXLInvalidDisks() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Make disk 5 and disk 8 have inconsistent disk uuid's.
+	formatConfigs[4].XL.Disk = getUUID()
+	formatConfigs[7].XL.Disk = getUUID()
+	return formatConfigs
+}
+
+// generates a invalid format.json Disk UUID in wrong order for XL backend.
+func genFormatXLInvalidDisksOrder() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Re order jbod for failure case.
+	var jbod1 = make([]string, 8)
+	for i, j := range jbod {
+		jbod1[i] = j
+	}
+	jbod1[1], jbod1[2] = jbod[2], jbod[1]
+	formatConfigs[2].XL.JBOD = jbod1
+	return formatConfigs
+}
+
+func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
+
+	var err error
+	xl := obj.(xlObjects)
+
+	err = obj.MakeBucket("bucket")
+	if err != nil {
+		return []StorageAPI{}, err
+	}
+
+	bucket := "bucket"
+	object := "object"
+
+	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
+	if err != nil {
+		return []StorageAPI{}, err
+	}
+
+	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
+	// created when minio starts
+	for i := 3; i <= 5; i++ {
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+			return []StorageAPI{}, err
+		}
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "tmp"); err != nil {
+			return []StorageAPI{}, err
+		}
+		if err = xl.storageDisks[i].DeleteFile(bucket, object+"/xl.json"); err != nil {
+			return []StorageAPI{}, err
+		}
+		if err = xl.storageDisks[i].DeleteFile(bucket, object+"/part.1"); err != nil {
+			return []StorageAPI{}, err
+		}
+		if err = xl.storageDisks[i].DeleteVol(bucket); err != nil {
+			return []StorageAPI{}, err
+		}
+	}
+
+	permutedStorageDisks := []StorageAPI{xl.storageDisks[1], xl.storageDisks[4],
+		xl.storageDisks[2], xl.storageDisks[8], xl.storageDisks[6], xl.storageDisks[7],
+		xl.storageDisks[0], xl.storageDisks[15], xl.storageDisks[13], xl.storageDisks[14],
+		xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
+		xl.storageDisks[5], xl.storageDisks[11]}
+
+	return permutedStorageDisks, nil
+
+}
+
+func TestFormatXLHealFreshDisks(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Error(err)
+	}
+
+	storageDisks, err := prepareFormatXLHealFreshDisks(obj)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Start healing disks
+	err = healFormatXLFreshDisks(storageDisks)
+	if err != nil {
+		t.Fatal("healing corrupted disk failed: ", err)
+	}
+
+	// Load again XL format.json to validate it
+	_, err = loadFormatXL(storageDisks)
+	if err != nil {
+		t.Fatal("loading healed disk failed: ", err)
+	}
+
+	// Clean all
+	removeRoots(fsDirs)
+}
+
+func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Error(err)
+	}
+
+	storageDisks, err := prepareFormatXLHealFreshDisks(obj)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 16; i++ {
+		d := storageDisks[i].(*posix)
+		storageDisks[i] = &naughtyDisk{disk: d, defaultErr: errDiskNotFound}
+	}
+
+	// Load again XL format.json to validate it
+	_, err = loadFormatXL(storageDisks)
+	if err == nil {
+		t.Fatal("loading format disk error")
+	}
+
+	storageDisks[3] = nil
+	err = healFormatXLFreshDisks(storageDisks)
+	if err != nil {
+		t.Fatal("didn't get nil when one disk is offline")
+	}
+
+	// Clean all
+	removeRoots(fsDirs)
+}
+
+// Simulate XL disks creation, delete some format.json and remove the content of
+// a given disk to test healing a corrupted disk
+func TestFormatXLHealCorruptedDisks(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	xl := obj.(xlObjects)
+
+	err = obj.MakeBucket("bucket")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := "bucket"
+	object := "object"
+
+	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Now, remove two format files.. Load them and reorder
+	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[11].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
+	// created when minio starts
+	if err = xl.storageDisks[10].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[10].DeleteFile(".minio.sys", "tmp"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[10].DeleteFile(bucket, object+"/xl.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[10].DeleteFile(bucket, object+"/part.1"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[10].DeleteVol(bucket); err != nil {
+		t.Fatal(err)
+	}
+
+	permutedStorageDisks := []StorageAPI{xl.storageDisks[1], xl.storageDisks[4],
+		xl.storageDisks[2], xl.storageDisks[8], xl.storageDisks[6], xl.storageDisks[7],
+		xl.storageDisks[0], xl.storageDisks[15], xl.storageDisks[13], xl.storageDisks[14],
+		xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
+		xl.storageDisks[5], xl.storageDisks[11]}
+
+	// Start healing disks
+	err = healFormatXLCorruptedDisks(permutedStorageDisks)
+	if err != nil {
+		t.Fatal("healing corrupted disk failed: ", err)
+	}
+
+	// Load again XL format.json to validate it
+	_, err = loadFormatXL(permutedStorageDisks)
+	if err != nil {
+		t.Fatal("loading healed disk failed: ", err)
+	}
+
+	// Clean all
+	removeRoots(fsDirs)
+}
+
+// Test on ReorderByInspection by simulating creating disks and removing
+// some of format.json
+func TestFormatXLReorderByInspection(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	xl := obj.(xlObjects)
+
+	err = obj.MakeBucket("bucket")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := "bucket"
+	object := "object"
+
+	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Now, remove two format files.. Load them and reorder
+	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[5].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+
+	permutedStorageDisks := []StorageAPI{xl.storageDisks[1], xl.storageDisks[4],
+		xl.storageDisks[2], xl.storageDisks[8], xl.storageDisks[6], xl.storageDisks[7],
+		xl.storageDisks[0], xl.storageDisks[15], xl.storageDisks[13], xl.storageDisks[14],
+		xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
+		xl.storageDisks[5], xl.storageDisks[11]}
+
+	permutedFormatConfigs, _ := loadAllFormats(permutedStorageDisks)
+
+	orderedDisks, err := reorderDisks(permutedStorageDisks, permutedFormatConfigs)
+	if err != nil {
+		t.Fatal("error reordering disks\n")
+	}
+
+	orderedDisks, err = reorderDisksByInspection(orderedDisks, permutedStorageDisks, permutedFormatConfigs)
+	if err != nil {
+		t.Fatal("failed to reorder disk by inspection")
+	}
+
+	// Check disks reordering
+	for i := 0; i <= 15; i++ {
+		if orderedDisks[i] == nil && i != 3 && i != 5 {
+			t.Fatal("should not be nil")
+		}
+		if orderedDisks[i] != nil && orderedDisks[i] != xl.storageDisks[i] {
+			t.Fatal("Disks were not ordered correctly.")
+		}
+	}
+
+	removeRoots(fsDirs)
+}
+
+// Wrapper for calling FormatXL tests - currently validates
+//  - valid format
+//  - unrecognized version number
+//  - unrecognized format tag
+//  - unrecognized xl version
+//  - wrong number of JBOD entries
+//  - invalid JBOD
+//  - invalid Disk uuid
+func TestFormatXL(t *testing.T) {
+	formatInputCases := [][]*formatConfigV1{
+		genFormatXLValid(),
+		genFormatXLInvalidVersion(),
+		genFormatXLInvalidFormat(),
+		genFormatXLInvalidXLVersion(),
+		genFormatXLInvalidJBODCount(),
+		genFormatXLInvalidJBOD(),
+		genFormatXLInvalidDisks(),
+		genFormatXLInvalidDisksOrder(),
+	}
+	testCases := []struct {
+		formatConfigs []*formatConfigV1
+		shouldPass    bool
+	}{
+		{
+			formatConfigs: formatInputCases[0],
+			shouldPass:    true,
+		},
+		{
+			formatConfigs: formatInputCases[1],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[2],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[3],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[4],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[5],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[6],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[7],
+			shouldPass:    false,
+		},
+	}
+
+	for i, testCase := range testCases {
+		err := checkFormatXL(testCase.formatConfigs)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass but failed with %s", i+1, err)
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail but passed instead", i+1)
+		}
+	}
+}
+
+// Tests uuid order verification function.
+func TestSavedUUIDOrder(t *testing.T) {
+	uuidTestCases := make([]struct {
+		uuid       string
+		shouldPass bool
+	}, 8)
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+		uuidTestCases[index].uuid = jbod[index]
+		uuidTestCases[index].shouldPass = true
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Re order jbod for failure case.
+	var jbod1 = make([]string, 8)
+	for i, j := range jbod {
+		jbod1[i] = j
+	}
+	jbod1[1], jbod1[2] = jbod[2], jbod[1]
+	formatConfigs[2].XL.JBOD = jbod1
+	uuidTestCases[1].shouldPass = false
+	uuidTestCases[2].shouldPass = false
+
+	for i, testCase := range uuidTestCases {
+		// Is uuid present on all JBOD ?.
+		if testCase.shouldPass != isSavedUUIDInOrder(testCase.uuid, formatConfigs) {
+			t.Errorf("Test %d: Expected to pass but failed", i+1)
+		}
+	}
+}
+
+// Test initFormatXL() when disks are expected to return errors
+func TestInitFormatXLErrors(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl := obj.(xlObjects)
+
+	testStorageDisks := make([]StorageAPI, 16)
+
+	// All disks API return disk not found
+	for i := 0; i < 16; i++ {
+		d := xl.storageDisks[i].(*posix)
+		testStorageDisks[i] = &naughtyDisk{disk: d, defaultErr: errDiskNotFound}
+	}
+	if err := initFormatXL(testStorageDisks); err != errDiskNotFound {
+		t.Fatal("Got a different error: ", err)
+	}
+
+	// All disks returns disk not found in the fourth call
+	for i := 0; i < 15; i++ {
+		d := xl.storageDisks[i].(*posix)
+		testStorageDisks[i] = &naughtyDisk{disk: d, defaultErr: errDiskNotFound, errors: map[int]error{0: nil, 1: nil, 2: nil}}
+	}
+	if err := initFormatXL(testStorageDisks); err != errDiskNotFound {
+		t.Fatal("Got a different error: ", err)
+	}
+
+	// All disks are nil (disk not found)
+	for i := 0; i < 15; i++ {
+		testStorageDisks[i] = nil
+	}
+	if err := initFormatXL(testStorageDisks); err != errDiskNotFound {
+		t.Fatal("Got a different error: ", err)
+	}
+
+	removeRoots(fsDirs)
+}
+
+// Test for reduceFormatErrs()
+func TestReduceFormatErrs(t *testing.T) {
+	// No error founds
+	if err := reduceFormatErrs([]error{nil, nil, nil, nil}, 4); err != nil {
+		t.Fatal("Err should be nil, found: ", err)
+	}
+	// One corrupted format
+	if err := reduceFormatErrs([]error{nil, nil, errCorruptedFormat, nil}, 4); err != errCorruptedFormat {
+		t.Fatal("Got a differnt error: ", err)
+	}
+	// All disks unformatted
+	if err := reduceFormatErrs([]error{errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, errUnformattedDisk}, 4); err != errUnformattedDisk {
+		t.Fatal("Got a differnt error: ", err)
+	}
+	// Some disks unformatted
+	if err := reduceFormatErrs([]error{nil, nil, errUnformattedDisk, errUnformattedDisk}, 4); err != errSomeDiskUnformatted {
+		t.Fatal("Got a differnt error: ", err)
+	}
+	// Some disks offline
+	if err := reduceFormatErrs([]error{nil, nil, errDiskNotFound, errUnformattedDisk}, 4); err != errSomeDiskOffline {
+		t.Fatal("Got a differnt error: ", err)
+	}
+}
+
+// Tests for genericFormatCheck()
+func TestGenericFormatCheck(t *testing.T) {
+	var errs []error
+	formatConfigs := genFormatXLInvalidJBOD()
+
+	// Some disks has corrupted formats, one faulty disk
+	errs = []error{nil, nil, errCorruptedFormat, errCorruptedFormat, errCorruptedFormat, errCorruptedFormat,
+		errCorruptedFormat, errFaultyDisk}
+	if err := genericFormatCheck(formatConfigs, errs); err != errCorruptedFormat {
+		t.Fatal("Got unexpected err: ", err)
+	}
+
+	// Many faulty disks
+	errs = []error{nil, nil, errFaultyDisk, errFaultyDisk, errFaultyDisk, errFaultyDisk,
+		errCorruptedFormat, errFaultyDisk}
+	if err := genericFormatCheck(formatConfigs, errs); err != errXLReadQuorum {
+		t.Fatal("Got unexpected err: ", err)
+	}
+
+	// All formats successfully loaded
+	errs = []error{nil, nil, nil, nil, nil, nil, nil, nil}
+	if err := genericFormatCheck(formatConfigs, errs); err == nil {
+		t.Fatalf("Should fail here")
+	}
+
+}
+
+func TestLoadFormatXLErrs(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl := obj.(xlObjects)
+
+	xl.storageDisks[11] = nil
+
+	// disk 12 returns faulty disk
+	posixDisk, ok := xl.storageDisks[12].(*posix)
+	if !ok {
+		t.Fatal("storage disk is not *posix type")
+	}
+	xl.storageDisks[10] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
+	if _, err = loadFormatXL(xl.storageDisks); err != errFaultyDisk {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+
+	removeRoots(fsDirs)
+
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+
+	// disks 0..10 returns disk not found
+	for i := 0; i <= 10; i++ {
+		posixDisk, ok := xl.storageDisks[i].(*posix)
+		if !ok {
+			t.Fatal("storage disk is not *posix type")
+		}
+		xl.storageDisks[i] = newNaughtyDisk(posixDisk, nil, errDiskNotFound)
+	}
+	if _, err = loadFormatXL(xl.storageDisks); err != errXLReadQuorum {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+
+	removeRoots(fsDirs)
+
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+
+	// disks 0..10 returns unformatted disk
+	for i := 0; i <= 10; i++ {
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err = loadFormatXL(xl.storageDisks); err != errUnformattedDisk {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+
+	removeRoots(fsDirs)
+
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+
+	// disks 0..15 returns is nil (disk not found)
+	for i := 0; i < 16; i++ {
+		xl.storageDisks[i] = nil
+	}
+	if _, err := loadFormatXL(xl.storageDisks); err != errDiskNotFound {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+
+	removeRoots(fsDirs)
+}
+
+// Tests for healFormatXLCorruptedDisks() with cases which lead to errors
+func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
+	// Everything is fine, should return nil
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl := obj.(xlObjects)
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Disks 0..15 are nil
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		xl.storageDisks[i] = nil
+	}
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// One disk returns Faulty Disk
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	posixDisk, ok := xl.storageDisks[0].(*posix)
+	if !ok {
+		t.Fatal("storage disk is not *posix type")
+	}
+	xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != errFaultyDisk {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// One disk is not found, heal corrupted disks should return nil
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	xl.storageDisks[0] = nil
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Remove format.json of all disks
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Corrupted format json in one disk
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		if err = xl.storageDisks[i].AppendFile(".minio.sys", "format.json", []byte("corrupted data")); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err == nil {
+		t.Fatal("Should get a json parsing error, ")
+	}
+	removeRoots(fsDirs)
+}
+
+// Tests for healFormatXLFreshDisks() with cases which lead to errors
+func TestHealFormatXLFreshDisksErrs(t *testing.T) {
+	// Everything is fine, should return nil
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl := obj.(xlObjects)
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Disks 0..15 are nil
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		xl.storageDisks[i] = nil
+	}
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// One disk returns Faulty Disk
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	posixDisk, ok := xl.storageDisks[0].(*posix)
+	if !ok {
+		t.Fatal("storage disk is not *posix type")
+	}
+	xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != errFaultyDisk {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// One disk is not found, heal corrupted disks should return nil
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	xl.storageDisks[0] = nil
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Remove format.json of all disks
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Remove format.json of all disks
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+}
+
+// Tests for isFormatFound()
+func TestIsFormatFound(t *testing.T) {
+	formats := genFormatXLValid()
+	if found := isFormatFound(formats); !found {
+		t.Fatal("isFormatFound() should not return false")
+	}
+	formats[0] = nil
+	if found := isFormatFound(formats); found {
+		t.Fatal("isFormatFound() should not return true")
+	}
+}
+
+// Tests for isFormatNotFound()
+func TestIsFormatNotFound(t *testing.T) {
+	formats := genFormatXLValid()
+	if found := isFormatNotFound(formats); found {
+		t.Fatal("isFormatFound() should not return true")
+	}
+	formats[0] = nil
+	if found := isFormatNotFound(formats); found {
+		t.Fatal("isFormatFound() should not return true")
+	}
+	for idx := range formats {
+		formats[idx] = nil
+	}
+	if found := isFormatNotFound(formats); !found {
+		t.Fatal("isFormatFound() should not return false")
+	}
+}

cmd/fs-createfile.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "io"
 

cmd/fs-v1-errors.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "errors"
 

cmd/fs-v1-metadata.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"encoding/json"
@@ -77,9 +77,9 @@ func (m *fsMetaV1) AddObjectPart(partNumber int, partName string, partETag strin
 }
 
 // readFSMetadata - returns the object metadata `fs.json` content.
-func readFSMetadata(disk StorageAPI, bucket, object string) (fsMeta fsMetaV1, err error) {
+func readFSMetadata(disk StorageAPI, bucket, filePath string) (fsMeta fsMetaV1, err error) {
 	// Read all `fs.json`.
-	buf, err := disk.ReadAll(bucket, path.Join(object, fsMetaJSONFile))
+	buf, err := disk.ReadAll(bucket, filePath)
 	if err != nil {
 		return fsMetaV1{}, err
 	}
@@ -93,12 +93,25 @@ func readFSMetadata(disk StorageAPI, bucket, object string) (fsMeta fsMetaV1, er
 	return fsMeta, nil
 }
 
+// Write fsMeta to fs.json or fs-append.json.
+func writeFSMetadata(disk StorageAPI, bucket, filePath string, fsMeta fsMetaV1) (err error) {
+	tmpPath := path.Join(tmpMetaPrefix, getUUID())
+	metadataBytes, err := json.Marshal(fsMeta)
+	if err != nil {
+		return err
+	}
+	if err = disk.AppendFile(minioMetaBucket, tmpPath, metadataBytes); err != nil {
+		return err
+	}
+	return disk.RenameFile(minioMetaBucket, tmpPath, bucket, filePath)
+}
+
 // newFSMetaV1 - initializes new fsMetaV1.
 func newFSMetaV1() (fsMeta fsMetaV1) {
 	fsMeta = fsMetaV1{}
 	fsMeta.Version = "1.0.0"
 	fsMeta.Format = "fs"
-	fsMeta.Minio.Release = minioReleaseTag
+	fsMeta.Minio.Release = ReleaseTag
 	return fsMeta
 }
 
@@ -131,17 +144,18 @@ func writeFSFormatData(storage StorageAPI, fsFormat formatConfigV1) error {
 	return nil
 }
 
-// writeFSMetadata - writes `fs.json` metadata, marshals fsMeta object into json
-// and saves it to disk.
-func writeFSMetadata(storage StorageAPI, bucket, path string, fsMeta fsMetaV1) error {
-	metadataBytes, err := json.Marshal(fsMeta)
-	if err != nil {
-		return err
+// Return if the part info in uploadedParts and completeParts are same.
+func isPartsSame(uploadedParts []objectPartInfo, completeParts []completePart) bool {
+	if len(uploadedParts) != len(completeParts) {
+		return false
 	}
-	if err = storage.AppendFile(bucket, path, metadataBytes); err != nil {
-		return err
+	for i := range completeParts {
+		if uploadedParts[i].Number != completeParts[i].PartNumber ||
+			uploadedParts[i].ETag != completeParts[i].ETag {
+			return false
+		}
 	}
-	return nil
+	return true
 }
 
 var extendedHeaders = []string{

cmd/fs-v1-metadata_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"os"

cmd/fs-v1-multipart-common.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"encoding/json"

cmd/fs-v1-multipart.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"crypto/md5"
@@ -68,8 +68,9 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 	}
 	var walkResultCh chan treeWalkResult
 	var endWalkCh chan struct{}
+	heal := false // true only for xl.ListObjectsHeal()
 	if maxUploads > 0 {
-		walkResultCh, endWalkCh = fs.listPool.Release(listParams{minioMetaBucket, recursive, multipartMarkerPath, multipartPrefixPath})
+		walkResultCh, endWalkCh = fs.listPool.Release(listParams{minioMetaBucket, recursive, multipartMarkerPath, multipartPrefixPath, heal})
 		if walkResultCh == nil {
 			endWalkCh = make(chan struct{})
 			isLeaf := fs.isMultipartUpload
@@ -144,7 +145,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 	if !eof {
 		// Save the go-routine state in the pool so that it can continue from where it left off on
 		// the next request.
-		fs.listPool.Set(listParams{bucket, recursive, result.NextKeyMarker, prefix}, walkResultCh, endWalkCh)
+		fs.listPool.Set(listParams{bucket, recursive, result.NextKeyMarker, prefix, heal}, walkResultCh, endWalkCh)
 	}
 
 	result.IsTruncated = !eof
@@ -218,6 +219,7 @@ func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[string]string) (uploadID string, err error) {
 	// Initialize `fs.json` values.
 	fsMeta := newFSMetaV1()
+
 	// Save additional metadata only if extended headers such as "X-Amz-Meta-" are set.
 	if hasExtendedHeader(meta) {
 		fsMeta.Meta = meta
@@ -233,17 +235,9 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st
 	if err = fs.writeUploadJSON(bucket, object, uploadID, initiated); err != nil {
 		return "", err
 	}
-	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
-	tempFSMetadataPath := path.Join(tmpMetaPrefix, getUUID()+"-"+fsMetaJSONFile)
-	if err = writeFSMetadata(fs.storage, minioMetaBucket, tempFSMetadataPath, fsMeta); err != nil {
-		return "", toObjectErr(err, minioMetaBucket, tempFSMetadataPath)
-	}
-	err = fs.storage.RenameFile(minioMetaBucket, tempFSMetadataPath, minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
-	if err != nil {
-		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempFSMetadataPath); dErr != nil {
-			return "", toObjectErr(dErr, minioMetaBucket, tempFSMetadataPath)
-		}
-		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+	fsMetaPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, fsMetaJSONFile)
+	if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
 	// Return success.
 	return uploadID, nil
@@ -270,6 +264,127 @@ func (fs fsObjects) NewMultipartUpload(bucket, object string, meta map[string]st
 	return fs.newMultipartUpload(bucket, object, meta)
 }
 
+// Returns if a new part can be appended to fsAppendDataFile.
+func partToAppend(fsMeta fsMetaV1, fsAppendMeta fsMetaV1) (part objectPartInfo, appendNeeded bool) {
+	if len(fsMeta.Parts) == 0 {
+		return
+	}
+	// As fsAppendMeta.Parts will be sorted len(fsAppendMeta.Parts) will naturally be the next part number
+	nextPartNum := len(fsAppendMeta.Parts) + 1
+	nextPartIndex := fsMeta.ObjectPartIndex(nextPartNum)
+	if nextPartIndex == -1 {
+		return
+	}
+	return fsMeta.Parts[nextPartIndex], true
+}
+
+// Returns metadata path for the file holding info about the parts that
+// have been appended to the "append-file"
+func getFSAppendMetaPath(uploadID string) string {
+	return path.Join(tmpMetaPrefix, uploadID+".json")
+}
+
+// Returns path for the append-file.
+func getFSAppendDataPath(uploadID string) string {
+	return path.Join(tmpMetaPrefix, uploadID+".data")
+}
+
+// Append parts to fsAppendDataFile.
+func appendParts(disk StorageAPI, bucket, object, uploadID string) {
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	// fs-append.json path
+	fsAppendMetaPath := getFSAppendMetaPath(uploadID)
+	// fs.json path
+	fsMetaPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, fsMetaJSONFile)
+
+	// Lock the uploadID so that no one modifies fs.json
+	nsMutex.RLock(minioMetaBucket, uploadIDPath)
+	fsMeta, err := readFSMetadata(disk, minioMetaBucket, fsMetaPath)
+	nsMutex.RUnlock(minioMetaBucket, uploadIDPath)
+	if err != nil {
+		return
+	}
+
+	// Lock fs-append.json so that there is no parallel append to the file.
+	nsMutex.Lock(minioMetaBucket, fsAppendMetaPath)
+	defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath)
+
+	fsAppendMeta, err := readFSMetadata(disk, minioMetaBucket, fsAppendMetaPath)
+	if err != nil {
+		if err != errFileNotFound {
+			return
+		}
+		fsAppendMeta = fsMeta
+		fsAppendMeta.Parts = nil
+	}
+
+	// Check if a part needs to be appended to
+	part, appendNeeded := partToAppend(fsMeta, fsAppendMeta)
+	if !appendNeeded {
+		return
+	}
+	// Hold write lock on the part so that there is no parallel upload on the part.
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(part.Number)))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(part.Number)))
+
+	// Proceed to append "part"
+	fsAppendDataPath := getFSAppendDataPath(uploadID)
+	tmpDataPath := path.Join(tmpMetaPrefix, getUUID())
+	if part.Number != 1 {
+		// Move it to tmp location before appending so that we don't leave inconsitent data
+		// if server crashes during append operation.
+		err = disk.RenameFile(minioMetaBucket, fsAppendDataPath, minioMetaBucket, tmpDataPath)
+		if err != nil {
+			return
+		}
+		// Delete fs-append.json so that we don't leave a stale file if server crashes
+		// when the part is being appended to the tmp file.
+		err = disk.DeleteFile(minioMetaBucket, fsAppendMetaPath)
+		if err != nil {
+			return
+		}
+	}
+	// Path to the part that needs to be appended.
+	partPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, part.Name)
+	offset := int64(0)
+	totalLeft := part.Size
+	buf := make([]byte, readSizeV1)
+	for totalLeft > 0 {
+		curLeft := int64(readSizeV1)
+		if totalLeft < readSizeV1 {
+			curLeft = totalLeft
+		}
+		var n int64
+		n, err = disk.ReadFile(minioMetaBucket, partPath, offset, buf[:curLeft])
+		if n > 0 {
+			if err = disk.AppendFile(minioMetaBucket, tmpDataPath, buf[:n]); err != nil {
+				return
+			}
+		}
+		if err != nil {
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				break
+			}
+			return
+		}
+		offset += n
+		totalLeft -= n
+	}
+	// All good, the part has been appended to the tmp file, rename it back.
+	if err = disk.RenameFile(minioMetaBucket, tmpDataPath, minioMetaBucket, fsAppendDataPath); err != nil {
+		return
+	}
+	fsAppendMeta.AddObjectPart(part.Number, part.Name, part.ETag, part.Size)
+	if err = writeFSMetadata(disk, minioMetaBucket, fsAppendMetaPath, fsAppendMeta); err != nil {
+		return
+	}
+	// If there are more parts that need to be appended to fsAppendDataFile
+	_, appendNeeded = partToAppend(fsMeta, fsAppendMeta)
+	if appendNeeded {
+		go appendParts(disk, bucket, object, uploadID)
+	}
+}
+
 // PutObjectPart - reads incoming data until EOF for the part file on
 // an ongoing multipart transaction. Internally incoming data is
 // written to '.minio/tmp' location and safely renamed to
@@ -297,12 +412,8 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		return "", InvalidUploadID{UploadID: uploadID}
 	}
 
-	// Hold write lock on the part so that there is no parallel upload on the part.
-	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(partID)))
-	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(partID)))
-
 	partSuffix := fmt.Sprintf("object%d", partID)
-	tmpPartPath := path.Join(tmpMetaPrefix, uploadID+"-"+partSuffix)
+	tmpPartPath := path.Join(tmpMetaPrefix, uploadID+"."+getUUID()+"."+partSuffix)
 
 	// Initialize md5 writer.
 	md5Writer := md5.New()
@@ -364,9 +475,10 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		return "", InvalidUploadID{UploadID: uploadID}
 	}
 
-	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
+	fsMetaPath := pathJoin(uploadIDPath, fsMetaJSONFile)
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, fsMetaPath)
 	if err != nil {
-		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+		return "", toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
 	fsMeta.AddObjectPart(partID, partSuffix, newMD5Hex, size)
 
@@ -378,18 +490,11 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		}
 		return "", toObjectErr(err, minioMetaBucket, partPath)
 	}
-	uploadIDPath = path.Join(mpartMetaPrefix, bucket, object, uploadID)
-	tempFSMetadataPath := path.Join(tmpMetaPrefix, getUUID()+"-"+fsMetaJSONFile)
-	if err = writeFSMetadata(fs.storage, minioMetaBucket, tempFSMetadataPath, fsMeta); err != nil {
-		return "", toObjectErr(err, minioMetaBucket, tempFSMetadataPath)
-	}
-	err = fs.storage.RenameFile(minioMetaBucket, tempFSMetadataPath, minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
-	if err != nil {
-		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempFSMetadataPath); dErr != nil {
-			return "", toObjectErr(dErr, minioMetaBucket, tempFSMetadataPath)
-		}
-		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+
+	if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
+	go appendParts(fs.storage, bucket, object, uploadID)
 	return newMD5Hex, nil
 }
 
@@ -399,10 +504,10 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) {
 	result := ListPartsInfo{}
 
-	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
-	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
+	fsMetaPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, fsMetaJSONFile)
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, fsMetaPath)
 	if err != nil {
-		return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, uploadIDPath)
+		return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
 	// Only parts with higher part numbers will be listed.
 	partIdx := fsMeta.ObjectPartIndex(partNumberMarker)
@@ -500,18 +605,18 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	// 1) no one aborts this multipart upload
 	// 2) no one does a parallel complete-multipart-upload on this
 	// multipart upload
-	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
-	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+	nsMutex.Lock(minioMetaBucket, uploadIDPath)
+	defer nsMutex.Unlock(minioMetaBucket, uploadIDPath)
 
 	if !fs.isUploadIDExists(bucket, object, uploadID) {
 		return "", InvalidUploadID{UploadID: uploadID}
 	}
 
-	// Read saved fs metadata for ongoing multipart.
-	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
-	if err != nil {
-		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
-	}
+	// fs-append.json path
+	fsAppendMetaPath := getFSAppendMetaPath(uploadID)
+	// Lock fs-append.json so that no parallel appendParts() is being done.
+	nsMutex.Lock(minioMetaBucket, fsAppendMetaPath)
+	defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath)
 
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5, err := completeMultipartMD5(parts...)
@@ -519,66 +624,83 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		return "", err
 	}
 
-	tempObj := path.Join(tmpMetaPrefix, uploadID+"-"+"part.1")
-
-	// Allocate staging buffer.
-	var buf = make([]byte, readSizeV1)
-
-	// Loop through all parts, validate them and then commit to disk.
-	for i, part := range parts {
-		partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
-		if partIdx == -1 {
-			return "", InvalidPart{}
-		}
-		if fsMeta.Parts[partIdx].ETag != part.ETag {
-			return "", BadDigest{}
-		}
-		// All parts except the last part has to be atleast 5MB.
-		if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
-			return "", PartTooSmall{
-				PartNumber: part.PartNumber,
-				PartSize:   fsMeta.Parts[partIdx].Size,
-				PartETag:   part.ETag,
-			}
-		}
-		// Construct part suffix.
-		partSuffix := fmt.Sprintf("object%d", part.PartNumber)
-		multipartPartFile := path.Join(mpartMetaPrefix, bucket, object, uploadID, partSuffix)
-		offset := int64(0)
-		totalLeft := fsMeta.Parts[partIdx].Size
-		for totalLeft > 0 {
-			curLeft := int64(readSizeV1)
-			if totalLeft < readSizeV1 {
-				curLeft = totalLeft
-			}
-			var n int64
-			n, err = fs.storage.ReadFile(minioMetaBucket, multipartPartFile, offset, buf[:curLeft])
-			if n > 0 {
-				if err = fs.storage.AppendFile(minioMetaBucket, tempObj, buf[:n]); err != nil {
-					return "", toObjectErr(err, minioMetaBucket, tempObj)
-				}
-			}
-			if err != nil {
-				if err == io.EOF || err == io.ErrUnexpectedEOF {
-					break
-				}
-				if err == errFileNotFound {
-					return "", InvalidPart{}
-				}
-				return "", toObjectErr(err, minioMetaBucket, multipartPartFile)
-			}
-			offset += n
-			totalLeft -= n
-		}
+	// Read saved fs metadata for ongoing multipart.
+	fsMetaPath := pathJoin(uploadIDPath, fsMetaJSONFile)
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, fsMetaPath)
+	if err != nil {
+		return "", toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
 
-	// Rename the file back to original location, if not delete the temporary object.
-	err = fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object)
-	if err != nil {
-		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempObj); dErr != nil {
-			return "", toObjectErr(dErr, minioMetaBucket, tempObj)
+	fsAppendMeta, err := readFSMetadata(fs.storage, minioMetaBucket, fsAppendMetaPath)
+	if err == nil && isPartsSame(fsAppendMeta.Parts, parts) {
+		fsAppendDataPath := getFSAppendDataPath(uploadID)
+		if err = fs.storage.RenameFile(minioMetaBucket, fsAppendDataPath, bucket, object); err != nil {
+			return "", toObjectErr(err, minioMetaBucket, fsAppendDataPath)
+		}
+		// Remove the append-file metadata file in tmp location as we no longer need it.
+		fs.storage.DeleteFile(minioMetaBucket, fsAppendMetaPath)
+	} else {
+		tempObj := path.Join(tmpMetaPrefix, uploadID+"-"+"part.1")
+
+		// Allocate staging buffer.
+		var buf = make([]byte, readSizeV1)
+
+		// Loop through all parts, validate them and then commit to disk.
+		for i, part := range parts {
+			partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
+			if partIdx == -1 {
+				return "", InvalidPart{}
+			}
+			if fsMeta.Parts[partIdx].ETag != part.ETag {
+				return "", BadDigest{}
+			}
+			// All parts except the last part has to be atleast 5MB.
+			if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
+				return "", PartTooSmall{
+					PartNumber: part.PartNumber,
+					PartSize:   fsMeta.Parts[partIdx].Size,
+					PartETag:   part.ETag,
+				}
+			}
+			// Construct part suffix.
+			partSuffix := fmt.Sprintf("object%d", part.PartNumber)
+			multipartPartFile := path.Join(mpartMetaPrefix, bucket, object, uploadID, partSuffix)
+			offset := int64(0)
+			totalLeft := fsMeta.Parts[partIdx].Size
+			for totalLeft > 0 {
+				curLeft := int64(readSizeV1)
+				if totalLeft < readSizeV1 {
+					curLeft = totalLeft
+				}
+				var n int64
+				n, err = fs.storage.ReadFile(minioMetaBucket, multipartPartFile, offset, buf[:curLeft])
+				if n > 0 {
+					if err = fs.storage.AppendFile(minioMetaBucket, tempObj, buf[:n]); err != nil {
+						return "", toObjectErr(err, minioMetaBucket, tempObj)
+					}
+				}
+				if err != nil {
+					if err == io.EOF || err == io.ErrUnexpectedEOF {
+						break
+					}
+					if err == errFileNotFound {
+						return "", InvalidPart{}
+					}
+					return "", toObjectErr(err, minioMetaBucket, multipartPartFile)
+				}
+				offset += n
+				totalLeft -= n
+			}
+		}
+
+		// Rename the file back to original location, if not delete the temporary object.
+		err = fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object)
+		if err != nil {
+			if dErr := fs.storage.DeleteFile(minioMetaBucket, tempObj); dErr != nil {
+				return "", toObjectErr(dErr, minioMetaBucket, tempObj)
+			}
+			return "", toObjectErr(err, bucket, object)
 		}
-		return "", toObjectErr(err, bucket, object)
 	}
 
 	// No need to save part info, since we have concatenated all parts.
@@ -591,13 +713,8 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		}
 		fsMeta.Meta["md5Sum"] = s3MD5
 
-		// Write the metadata to a temp file and rename it to the actual location.
-		tmpMetaPath := path.Join(tmpMetaPrefix, getUUID())
-		if err = writeFSMetadata(fs.storage, minioMetaBucket, tmpMetaPath, fsMeta); err != nil {
-			return "", toObjectErr(err, bucket, object)
-		}
-		fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
-		if err = fs.storage.RenameFile(minioMetaBucket, tmpMetaPath, minioMetaBucket, fsMetaPath); err != nil {
+		fsMetaPath = path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
+		if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
 			return "", toObjectErr(err, bucket, object)
 		}
 	}
@@ -707,6 +824,11 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 		return InvalidUploadID{UploadID: uploadID}
 	}
 
+	fsAppendMetaPath := getFSAppendMetaPath(uploadID)
+	// Lock fs-append.json so that no parallel appendParts() is being done.
+	nsMutex.Lock(minioMetaBucket, fsAppendMetaPath)
+	defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath)
+
 	err := fs.abortMultipartUpload(bucket, object, uploadID)
 	return err
 }

cmd/fs-v1.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"crypto/md5"
@@ -23,7 +23,6 @@ import (
 	"io"
 	"os"
 	"path"
-	"path/filepath"
 	"sort"
 	"strings"
 
@@ -325,7 +324,7 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
 	if err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
-	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, object))
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile))
 	if err != nil && err != errFileNotFound {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
@@ -336,7 +335,7 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
 
 	// Guess content-type from the extension if possible.
 	if fsMeta.Meta["content-type"] == "" {
-		if objectExt := filepath.Ext(object); objectExt != "" {
+		if objectExt := path.Ext(object); objectExt != "" {
 			if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
 				fsMeta.Meta["content-type"] = content.ContentType
 			}
@@ -369,6 +368,10 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 			Object: object,
 		}
 	}
+	// No metadata is set, allocate a new one.
+	if metadata == nil {
+		metadata = make(map[string]string)
+	}
 
 	uniqueID := getUUID()
 
@@ -419,10 +422,9 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 	}
 
 	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
-	// md5Hex representation.
-	var md5Hex string
-	if len(metadata) != 0 {
-		md5Hex = metadata["md5Sum"]
+	// Update the md5sum if not set with the newly calculated one.
+	if len(metadata["md5Sum"]) == 0 {
+		metadata["md5Sum"] = newMD5Hex
 	}
 
 	// Validate if payload is valid.
@@ -435,6 +437,8 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 		}
 	}
 
+	// md5Hex representation.
+	md5Hex := metadata["md5Sum"]
 	if md5Hex != "" {
 		if newMD5Hex != md5Hex {
 			// MD5 mismatch, delete the temporary object.
@@ -456,13 +460,8 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 		fsMeta := newFSMetaV1()
 		fsMeta.Meta = metadata
 
-		// Write the metadata to a temp file and rename it to the actual location.
-		tmpMetaPath := path.Join(tmpMetaPrefix, getUUID())
 		fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
-		if err = writeFSMetadata(fs.storage, minioMetaBucket, tmpMetaPath, fsMeta); err != nil {
-			return "", toObjectErr(err, bucket, object)
-		}
-		if err = fs.storage.RenameFile(minioMetaBucket, tmpMetaPath, minioMetaBucket, fsMetaPath); err != nil {
+		if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
 			return "", toObjectErr(err, bucket, object)
 		}
 	}
@@ -505,7 +504,9 @@ func isBucketExist(storage StorageAPI, bucketName string) bool {
 	return true
 }
 
-func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+// ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
+// state for future re-entrant list requests.
+func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
 	// Convert entry to FileInfo
 	entryToFileInfo := func(entry string) (fileInfo FileInfo, err error) {
 		if strings.HasSuffix(entry, slashSeparator) {
@@ -517,8 +518,16 @@ func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey
 		if fileInfo, err = fs.storage.StatFile(bucket, entry); err != nil {
 			return
 		}
+		fsMeta, mErr := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, entry, fsMetaJSONFile))
+		if mErr != nil && mErr != errFileNotFound {
+			return FileInfo{}, mErr
+		}
+		if len(fsMeta.Meta) == 0 {
+			fsMeta.Meta = make(map[string]string)
+		}
 		// Object name needs to be full path.
 		fileInfo.Name = entry
+		fileInfo.MD5Sum = fsMeta.Meta["md5Sum"]
 		return
 	}
 
@@ -574,7 +583,8 @@ func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey
 		recursive = false
 	}
 
-	walkResultCh, endWalkCh := fs.listPool.Release(listParams{bucket, recursive, marker, prefix})
+	heal := false // true only for xl.ListObjectsHeal()
+	walkResultCh, endWalkCh := fs.listPool.Release(listParams{bucket, recursive, marker, prefix, heal})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
 		isLeaf := func(bucket, object string) bool {
@@ -616,7 +626,7 @@ func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey
 		}
 		i++
 	}
-	params := listParams{bucket, recursive, nextMarker, prefix}
+	params := listParams{bucket, recursive, nextMarker, prefix, heal}
 	if !eof {
 		fs.listPool.Set(params, walkResultCh, endWalkCh)
 	}
@@ -639,7 +649,12 @@ func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey
 	return result, nil
 }
 
-// ListObjects - list all objects.
-func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
-	return fs.listObjects(bucket, prefix, marker, delimiter, maxKeys)
+// HealObject - no-op for fs. Valid only for XL.
+func (fs fsObjects) HealObject(bucket, object string) error {
+	return NotImplemented{}
+}
+
+// HealListObjects - list objects for healing. Valid only for XL
+func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+	return ListObjectsInfo{}, NotImplemented{}
 }

cmd/fs-v1_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"os"

cmd/generic-handlers.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"net/http"
@@ -77,7 +77,7 @@ func setBrowserRedirectHandler(h http.Handler) http.Handler {
 
 func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Re-direction handled specifically for browsers.
-	if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
+	if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && !isRequestSignatureV4(r) {
 		// '/' is redirected to 'locationPrefix/'
 		// '/webrpc' is redirected to 'locationPrefix/webrpc'
 		// '/login' is redirected to 'locationPrefix/login'
@@ -198,9 +198,10 @@ func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			writeErrorResponse(w, r, apiErr, r.URL.Path)
 			return
 		}
-		// Verify if the request date header is more than 5minutes
-		// late, reject such clients.
-		if time.Now().UTC().Sub(amzDate)/time.Minute > time.Duration(5)*time.Minute {
+		// Verify if the request date header is shifted by less than maxSkewTime parameter in the past
+		// or in the future, reject request otherwise.
+		curTime := time.Now().UTC()
+		if curTime.Sub(amzDate) > maxSkewTime || amzDate.Sub(curTime) > maxSkewTime {
 			writeErrorResponse(w, r, ErrRequestTimeTooSkewed, r.URL.Path)
 			return
 		}

cmd/globals.go

@@ -14,9 +14,11 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
+	"time"
+
 	"github.com/fatih/color"
 	"github.com/minio/minio/pkg/objcache"
 )
@@ -28,7 +30,7 @@ const (
 
 // minio configuration related constants.
 const (
-	globalMinioConfigVersion = "6"
+	globalMinioConfigVersion = "7"
 	globalMinioConfigDir     = ".minio"
 	globalMinioCertsDir      = "certs"
 	globalMinioCertFile      = "public.crt"
@@ -58,6 +60,11 @@ var (
 	maxFormFieldSize = int64(1024 * 1024)
 )
 
+var (
+	// The maximum allowed difference between the request generation time and the server processing time
+	maxSkewTime = 15 * time.Minute
+)
+
 // global colors.
 var (
 	colorBlue = color.New(color.FgBlue).SprintfFunc()

cmd/handler-utils.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"io"

cmd/handler-utils_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/host-to-ip.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"net"

cmd/host-to-ip_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "testing"
 

cmd/httprange.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"fmt"

cmd/httprange_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "testing"
 

cmd/leak-detect_test.go

@@ -5,7 +5,7 @@
 // Package leaktest provides tools to detect leaked goroutines in tests.
 // To use it, call "defer leaktest.AfterTest(t)()" at the beginning of each
 // test that may use goroutines.
-package main
+package cmd
 
 import (
 	"runtime"

cmd/logger-console-hook.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"io/ioutil"

cmd/logger-file-hook.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"fmt"

cmd/logger-syslog-hook_nix.go

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"fmt"

cmd/logger-syslog-hook_windows.go

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 type syslogLogger struct {
 	Enable bool   `json:"enable"`

cmd/logger.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bufio"
@@ -56,7 +56,7 @@ func stackInfo() string {
 	stackBuf.ReadFrom(rawStack)
 
 	// Strip GOPATH of the build system and return.
-	return strings.Replace(stackBuf.String(), minioGOPATH+"/src/", "", -1)
+	return strings.Replace(stackBuf.String(), GOPATH+"/src/", "", -1)
 }
 
 // errorIf synonymous with fatalIf but doesn't exit on error != nil

cmd/logger_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/main.go

@@ -0,0 +1,191 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/minio/cli"
+	"github.com/minio/mc/pkg/console"
+)
+
+var (
+	// global flags for minio.
+	minioFlags = []cli.Flag{
+		cli.BoolFlag{
+			Name:  "help, h",
+			Usage: "Show help.",
+		},
+	}
+)
+
+// Help template for minio.
+var minioHelpTemplate = `NAME:
+  {{.Name}} - {{.Usage}}
+
+DESCRIPTION:
+  {{.Description}}
+
+USAGE:
+  minio {{if .Flags}}[flags] {{end}}command{{if .Flags}}{{end}} [arguments...]
+
+COMMANDS:
+  {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
+  {{end}}{{if .Flags}}
+FLAGS:
+  {{range .Flags}}{{.}}
+  {{end}}{{end}}
+VERSION:
+  ` + Version +
+	`{{ "\n"}}`
+
+// init - check the environment before main starts
+func init() {
+	// Check if minio was compiled using a supported version of Golang.
+	checkGoVersion()
+
+	// Set global trace flag.
+	globalTrace = os.Getenv("MINIO_TRACE") == "1"
+}
+
+func migrate() {
+	// Migrate config file
+	migrateConfig()
+
+	// Migrate other configs here.
+}
+
+func enableLoggers() {
+	// Enable all loggers here.
+	enableConsoleLogger()
+	enableFileLogger()
+	// Add your logger here.
+}
+
+func findClosestCommands(command string) []string {
+	var closestCommands []string
+	for _, value := range commandsTree.PrefixMatch(command) {
+		closestCommands = append(closestCommands, value.(string))
+	}
+	sort.Strings(closestCommands)
+	// Suggest other close commands - allow missed, wrongly added and
+	// even transposed characters
+	for _, value := range commandsTree.walk(commandsTree.root) {
+		if sort.SearchStrings(closestCommands, value.(string)) < len(closestCommands) {
+			continue
+		}
+		// 2 is arbitrary and represents the max
+		// allowed number of typed errors
+		if DamerauLevenshteinDistance(command, value.(string)) < 2 {
+			closestCommands = append(closestCommands, value.(string))
+		}
+	}
+	return closestCommands
+}
+
+func registerApp() *cli.App {
+	// Register all commands.
+	registerCommand(serverCmd)
+	registerCommand(versionCmd)
+	registerCommand(updateCmd)
+	registerCommand(controlCmd)
+
+	// Set up app.
+	app := cli.NewApp()
+	app.Name = "Minio"
+	app.Author = "Minio.io"
+	app.Usage = "Cloud Storage Server."
+	app.Description = `Minio is an Amazon S3 compatible object storage server. Use it to store photos, videos, VMs, containers, log files, or any blob of data as objects.`
+	app.Flags = append(minioFlags, globalFlags...)
+	app.Commands = commands
+	app.CustomAppHelpTemplate = minioHelpTemplate
+	app.CommandNotFound = func(ctx *cli.Context, command string) {
+		msg := fmt.Sprintf("‘%s’ is not a minio sub-command. See ‘minio --help’.", command)
+		closestCommands := findClosestCommands(command)
+		if len(closestCommands) > 0 {
+			msg += fmt.Sprintf("\n\nDid you mean one of these?\n")
+			for _, cmd := range closestCommands {
+				msg += fmt.Sprintf("        ‘%s’\n", cmd)
+			}
+		}
+		console.Fatalln(msg)
+	}
+	return app
+}
+
+// Verify main command syntax.
+func checkMainSyntax(c *cli.Context) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		console.Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
+	}
+	if configPath == "" {
+		console.Fatalln("Config folder cannot be empty, please specify --config-dir <foldername>.")
+	}
+}
+
+// Main main for minio server.
+func Main() {
+	app := registerApp()
+	app.Before = func(c *cli.Context) error {
+		// Sets new config folder.
+		setGlobalConfigPath(c.GlobalString("config-dir"))
+
+		// Valid input arguments to main.
+		checkMainSyntax(c)
+
+		// Migrate any old version of config / state files to newer format.
+		migrate()
+
+		// Initialize config.
+		err := initConfig()
+		fatalIf(err, "Unable to initialize minio config.")
+
+		// Enable all loggers by now.
+		enableLoggers()
+
+		// Set global quiet flag.
+		globalQuiet = c.Bool("quiet") || c.GlobalBool("quiet")
+
+		// Do not print update messages, if quiet flag is set.
+		if !globalQuiet {
+			if strings.HasPrefix(ReleaseTag, "RELEASE.") && c.Args().Get(0) != "update" {
+				updateMsg, _, err := getReleaseUpdate(minioUpdateStableURL, time.Second*1)
+				if err != nil {
+					// Ignore all network related errors.
+					return nil
+				}
+				if updateMsg.Update {
+					console.Println(updateMsg)
+				}
+			}
+		}
+		return nil
+	}
+
+	// Start profiler if env is set.
+	if profiler := os.Getenv("MINIO_PROFILER"); profiler != "" {
+		globalProfiler = startProfiler(profiler)
+	}
+
+	// Run the app - exit on error.
+	app.RunAndExitOnError()
+}

cmd/main_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"testing"

cmd/namespace-lock.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"errors"

cmd/namespace-lock_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "testing"
 

cmd/naughty-disk_test.go

@@ -0,0 +1,123 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// naughtyDisk wraps a POSIX disk and returns programmed errors
+// specified by the developer. The purpose is to simulate errors
+// that are hard to simulate in practise like DiskNotFound.
+// Programmed errors are stored in errors field.
+type naughtyDisk struct {
+	// The real disk
+	disk *posix
+	// Programmed errors: API call number => error to return
+	errors map[int]error
+	// The error to return when no error value is programmed
+	defaultErr error
+	// The current API call number
+	callNR int
+}
+
+func newNaughtyDisk(d *posix, errs map[int]error, defaultErr error) *naughtyDisk {
+	return &naughtyDisk{disk: d, errors: errs, defaultErr: defaultErr}
+}
+
+func (d *naughtyDisk) calcError() (err error) {
+	d.callNR++
+	if err, ok := d.errors[d.callNR]; ok {
+		return err
+	}
+	if d.defaultErr != nil {
+		return d.defaultErr
+	}
+	return nil
+}
+
+func (d *naughtyDisk) MakeVol(volume string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.MakeVol(volume)
+}
+
+func (d *naughtyDisk) ListVols() (vols []VolInfo, err error) {
+	if err := d.calcError(); err != nil {
+		return nil, err
+	}
+	return d.disk.ListVols()
+}
+
+func (d *naughtyDisk) StatVol(volume string) (volInfo VolInfo, err error) {
+	if err := d.calcError(); err != nil {
+		return VolInfo{}, err
+	}
+	return d.disk.StatVol(volume)
+}
+func (d *naughtyDisk) DeleteVol(volume string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.DeleteVol(volume)
+}
+
+func (d *naughtyDisk) ListDir(volume, path string) (entries []string, err error) {
+	if err := d.calcError(); err != nil {
+		return []string{}, err
+	}
+	return d.disk.ListDir(volume, path)
+}
+
+func (d *naughtyDisk) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) {
+	if err := d.calcError(); err != nil {
+		return 0, err
+	}
+	return d.disk.ReadFile(volume, path, offset, buf)
+}
+
+func (d *naughtyDisk) AppendFile(volume, path string, buf []byte) error {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.AppendFile(volume, path, buf)
+}
+
+func (d *naughtyDisk) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.RenameFile(srcVolume, srcPath, dstVolume, dstPath)
+}
+
+func (d *naughtyDisk) StatFile(volume string, path string) (file FileInfo, err error) {
+	if err := d.calcError(); err != nil {
+		return FileInfo{}, err
+	}
+	return d.disk.StatFile(volume, path)
+}
+
+func (d *naughtyDisk) DeleteFile(volume string, path string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.DeleteFile(volume, path)
+}
+
+func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error) {
+	if err := d.calcError(); err != nil {
+		return nil, err
+	}
+	return d.disk.ReadAll(volume, path)
+}

cmd/notifiers.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"errors"
@@ -114,7 +114,7 @@ func isElasticQueue(sqsArn arnSQS) bool {
 // Match function matches wild cards in 'pattern' for events.
 func eventMatch(eventType string, events []string) (ok bool) {
 	for _, event := range events {
-		ok = wildcard.Match(event, eventType)
+		ok = wildcard.MatchSimple(event, eventType)
 		if ok {
 			break
 		}

cmd/notifiers_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "testing"
 

cmd/notify-amqp.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"io/ioutil"

cmd/notify-elasticsearch.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"errors"

cmd/notify-redis.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"io/ioutil"

cmd/object-api-getobject_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/object-api-getobjectinfo_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/object-api-listobjects_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/object-api-multipart_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"

cmd/object-api-putobject_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import (
 	"bytes"